Merge remote-tracking branch 'upstream/dev' into feature-response-headers

# Conflicts:
#	server/db/pg/schema/schema.ts
#	server/lib/blueprints/proxyResources.ts
#	server/routers/resource/getResource.ts
#	src/app/[orgId]/settings/resources/proxy/[niceId]/proxy/page.tsx
#	src/components/HealthCheckCredenza.tsx
This commit is contained in:
Julian van der Horst
2026-07-07 11:13:15 +02:00
695 changed files with 67748 additions and 22904 deletions
+41 -15
View File
@@ -5,6 +5,7 @@ import { and, eq, inArray } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import logger from "@server/logger";
export enum ActionsEnum {
createOrgUser = "createOrgUser",
@@ -20,6 +21,7 @@ export enum ActionsEnum {
getSite = "getSite",
listSites = "listSites",
updateSite = "updateSite",
restartSite = "restartSite",
resetSiteBandwidth = "resetSiteBandwidth",
reGenerateSecret = "reGenerateSecret",
createResource = "createResource",
@@ -158,7 +160,27 @@ export enum ActionsEnum {
createHealthCheck = "createHealthCheck",
updateHealthCheck = "updateHealthCheck",
deleteHealthCheck = "deleteHealthCheck",
listHealthChecks = "listHealthChecks"
listHealthChecks = "listHealthChecks",
createBrowserGatewayTarget = "createBrowserGatewayTarget",
updateBrowserGatewayTarget = "updateBrowserGatewayTarget",
deleteBrowserGatewayTarget = "deleteBrowserGatewayTarget",
getBrowserGatewayTarget = "getBrowserGatewayTarget",
listBrowserGatewayTargets = "listBrowserGatewayTargets",
listResourcePolicies = "listResourcePolicies",
getResourcePolicy = "getResourcePolicy",
createResourcePolicy = "createResourcePolicy",
updateResourcePolicy = "updateResourcePolicy",
deleteResourcePolicy = "deleteResourcePolicy",
listResourcePolicyRoles = "listResourcePolicyRoles",
setResourcePolicyRoles = "setResourcePolicyRoles",
listResourcePolicyUsers = "listResourcePolicyUsers",
setResourcePolicyUsers = "setResourcePolicyUsers",
setResourcePolicyPassword = "setResourcePolicyPassword",
setResourcePolicyPincode = "setResourcePolicyPincode",
setResourcePolicyHeaderAuth = "setResourcePolicyHeaderAuth",
setResourcePolicyWhitelist = "setResourcePolicyWhitelist",
setResourcePolicyRules = "setResourcePolicyRules",
createOrgWideLauncherView = "createOrgWideLauncherView"
}
export async function checkUserActionPermission(
@@ -191,6 +213,23 @@ export async function checkUserActionPermission(
}
}
// If no direct permission, check role-based permission (any of user's roles)
const roleActionPermission = await db
.select()
.from(roleActions)
.where(
and(
eq(roleActions.actionId, actionId),
inArray(roleActions.roleId, userOrgRoleIds),
eq(roleActions.orgId, req.userOrgId!)
)
)
.limit(1);
if (roleActionPermission.length > 0) {
return true;
}
// Check if the user has direct permission for the action in the current org
const userActionPermission = await db
.select()
@@ -208,20 +247,7 @@ export async function checkUserActionPermission(
return true;
}
// If no direct permission, check role-based permission (any of user's roles)
const roleActionPermission = await db
.select()
.from(roleActions)
.where(
and(
eq(roleActions.actionId, actionId),
inArray(roleActions.roleId, userOrgRoleIds),
eq(roleActions.orgId, req.userOrgId!)
)
)
.limit(1);
return roleActionPermission.length > 0;
return false;
} catch (error) {
console.error("Error checking user action permission:", error);
throw createHttpError(
+96 -24
View File
@@ -1,6 +1,12 @@
import { db } from "@server/db";
import { and, eq, inArray } from "drizzle-orm";
import { roleResources, userResources } from "@server/db";
import { and, eq, inArray, isNull, or } from "drizzle-orm";
import {
rolePolicies,
roleResources,
resources,
userPolicies,
userResources
} from "@server/db";
export async function canUserAccessResource({
userId,
@@ -11,9 +17,14 @@ export async function canUserAccessResource({
resourceId: number;
roleIds: number[];
}): Promise<boolean> {
const roleResourceAccess =
const [
roleResourceAccess,
rolePolicyAccess,
userResourceAccess,
userPolicyAccess
] = await Promise.all([
roleIds.length > 0
? await db
? db
.select()
.from(roleResources)
.where(
@@ -23,26 +34,87 @@ export async function canUserAccessResource({
)
)
.limit(1)
: [];
if (roleResourceAccess.length > 0) {
return true;
}
const userResourceAccess = await db
.select()
.from(userResources)
.where(
and(
eq(userResources.userId, userId),
eq(userResources.resourceId, resourceId)
: [],
roleIds.length > 0
? db
.select({
roleId: rolePolicies.roleId,
resourcePolicyId: rolePolicies.resourcePolicyId
})
.from(rolePolicies)
.innerJoin(
resources,
// Shared policy wins; only use default policy when no shared
// policy is assigned to the resource.
or(
eq(
resources.resourcePolicyId,
rolePolicies.resourcePolicyId
),
and(
isNull(resources.resourcePolicyId),
eq(
resources.defaultResourcePolicyId,
rolePolicies.resourcePolicyId
)
)
)
)
.where(
and(
eq(resources.resourceId, resourceId),
inArray(rolePolicies.roleId, roleIds)
)
)
.limit(1)
: [],
db
.select()
.from(userResources)
.where(
and(
eq(userResources.userId, userId),
eq(userResources.resourceId, resourceId)
)
)
)
.limit(1);
.limit(1),
db
.select({
userId: userPolicies.userId,
resourcePolicyId: userPolicies.resourcePolicyId
})
.from(userPolicies)
.innerJoin(
resources,
// Shared policy wins; only use default policy when no shared
// policy is assigned to the resource.
or(
eq(
resources.resourcePolicyId,
userPolicies.resourcePolicyId
),
and(
isNull(resources.resourcePolicyId),
eq(
resources.defaultResourcePolicyId,
userPolicies.resourcePolicyId
)
)
)
)
.where(
and(
eq(resources.resourceId, resourceId),
eq(userPolicies.userId, userId)
)
)
.limit(1)
]);
if (userResourceAccess.length > 0) {
return true;
}
return false;
return (
roleResourceAccess.length > 0 ||
rolePolicyAccess.length > 0 ||
userResourceAccess.length > 0 ||
userPolicyAccess.length > 0
);
}
+40 -1
View File
@@ -12,7 +12,7 @@ import {
users
} from "@server/db";
import { db } from "@server/db";
import { eq, inArray } from "drizzle-orm";
import { and, eq, inArray, ne } from "drizzle-orm";
import config from "@server/lib/config";
import type { RandomReader } from "@oslojs/crypto/random";
import { generateRandomString } from "@oslojs/crypto/random";
@@ -136,6 +136,45 @@ export async function invalidateAllSessions(userId: string): Promise<void> {
}
}
export async function invalidateAllSessionsExceptCurrent(
userId: string,
currentSessionId: string
): Promise<void> {
try {
await db.transaction(async (trx) => {
const userSessions = await trx
.select()
.from(sessions)
.where(
and(
eq(sessions.userId, userId),
ne(sessions.sessionId, currentSessionId)
)
);
if (userSessions.length > 0) {
await trx.delete(resourceSessions).where(
inArray(
resourceSessions.userSessionId,
userSessions.map((s) => s.sessionId)
)
);
}
await trx
.delete(sessions)
.where(
and(
eq(sessions.userId, userId),
ne(sessions.sessionId, currentSessionId)
)
);
});
} catch (e) {
logger.error("Failed to invalidate user sessions except current", e);
}
}
export function serializeSessionCookie(
token: string,
isSecure: boolean,
+10 -1
View File
@@ -19,6 +19,9 @@ export async function createResourceSession(opts: {
userSessionId?: string | null;
whitelistId?: number | null;
accessTokenId?: string | null;
policyPasswordId?: number | null;
policyPincodeId?: number | null;
policyWhitelistId?: number | null;
doNotExtend?: boolean;
expiresAt?: number | null;
sessionLength?: number | null;
@@ -28,7 +31,10 @@ export async function createResourceSession(opts: {
!opts.pincodeId &&
!opts.whitelistId &&
!opts.accessTokenId &&
!opts.userSessionId
!opts.userSessionId &&
!opts.policyPasswordId &&
!opts.policyPincodeId &&
!opts.policyWhitelistId
) {
throw new Error("Auth method must be provided");
}
@@ -49,6 +55,9 @@ export async function createResourceSession(opts: {
whitelistId: opts.whitelistId || null,
doNotExtend: opts.doNotExtend || false,
accessTokenId: opts.accessTokenId || null,
policyPasswordId: opts.policyPasswordId || null,
policyPincodeId: opts.policyPincodeId || null,
policyWhitelistId: opts.policyWhitelistId || null,
isRequestToken: opts.isRequestToken || false,
userSessionId: opts.userSessionId || null,
issuedAt: new Date().getTime()
+7 -4
View File
@@ -795,10 +795,13 @@ export const COUNTRIES = [
name: "Serbia",
code: "RS"
},
{
name: "Serbia and Montenegro",
code: "CS"
},
// Removed as this is a deprecated ISO country code, not supported anymore
// Also the individual flags for Serbia & Montenegro are already included in the list
// more details: https://en.wikipedia.org/wiki/ISO_3166-2:CS
// {
// name: "Serbia and Montenegro",
// code: "CS"
// },
{
name: "Seychelles",
code: "SC"
+36 -1
View File
@@ -1,6 +1,12 @@
import { join } from "path";
import { readFileSync } from "fs";
import { clients, db, resources, siteResources } from "@server/db";
import {
clients,
db,
resourcePolicies,
resources,
siteResources
} from "@server/db";
import { randomInt } from "crypto";
import { exitNodes, sites } from "@server/db";
import { eq, and } from "drizzle-orm";
@@ -107,6 +113,35 @@ export async function getUniqueResourceName(orgId: string): Promise<string> {
}
}
export async function getUniqueResourcePolicyName(
orgId: string
): Promise<string> {
let loops = 0;
while (true) {
if (loops > 100) {
throw new Error("Could not generate a unique name");
}
const name = generateName();
const policyCount = await db
.select({
niceId: resourcePolicies.niceId,
orgId: resourcePolicies.orgId
})
.from(resourcePolicies)
.where(
and(
eq(resourcePolicies.niceId, name),
eq(resourcePolicies.orgId, orgId)
)
);
if (policyCount.length === 0) {
return name;
}
loops++;
}
}
export async function getUniqueSiteResourceName(
orgId: string
): Promise<string> {
+1 -1
View File
@@ -87,7 +87,7 @@ function createDb() {
export const db = createDb();
export default db;
export const primaryDb = db.$primary as typeof db; // is this typeof a problem - techincally they are different types
export const primaryDb = db.$primary as typeof db; // is this typeof a problem - technically they are different types
export type Transaction = Parameters<
Parameters<(typeof db)["transaction"]>[0]
>[0];
+1
View File
@@ -4,3 +4,4 @@ export * from "./safeRead";
export * from "./schema/schema";
export * from "./schema/privateSchema";
export * from "./migrate";
export { alias } from "drizzle-orm/pg-core";
+3 -4
View File
@@ -2,7 +2,7 @@ import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres";
import { readConfigFile } from "@server/lib/readConfigFile";
import { withReplicas } from "drizzle-orm/pg-core";
import { build } from "@server/build";
import { db as mainDb, primaryDb as mainPrimaryDb } from "./driver";
import { db as mainDb } from "./driver";
import { createPool } from "./poolConfig";
function createLogsDb() {
@@ -63,8 +63,7 @@ function createLogsDb() {
})
);
} else {
const maxReplicaConnections =
poolConfig?.max_replica_connections || 20;
const maxReplicaConnections = poolConfig?.max_replica_connections || 20;
for (const conn of replicaConnections) {
const replicaPool = createPool(
conn.connection_string,
@@ -91,4 +90,4 @@ function createLogsDb() {
export const logsDb = createLogsDb();
export default logsDb;
export const primaryLogsDb = logsDb.$primary;
export const primaryLogsDb = logsDb.$primary;
+14 -5
View File
@@ -1,5 +1,4 @@
import { Pool, PoolConfig } from "pg";
import logger from "@server/logger";
export function createPoolConfig(
connectionString: string,
@@ -19,7 +18,17 @@ export function createPoolConfig(
keepAliveInitialDelayMillis: 10000, // send first keepalive after 10s of idle
// Allow connections to be released and recreated more aggressively
// to avoid stale connections building up
allowExitOnIdle: false
allowExitOnIdle: false,
// Disable JIT compilation for this connection. Our hot-path queries
// (e.g. resource-by-domain lookups) join many tables but only ever
// return a handful of rows. When planner row estimates drift (e.g.
// due to autovacuum lag under write-heavy load), Postgres decides
// these plans are expensive enough to JIT-compile, which can add
// multiple seconds of pure compilation overhead per query and
// saturate the connection pool. JIT never pays off for these
// short-lived OLTP queries, so it's disabled outright rather than
// relying on statistics staying fresh.
options: "-c jit=off"
};
}
@@ -27,7 +36,7 @@ export function attachPoolErrorHandlers(pool: Pool, label: string): void {
pool.on("error", (err) => {
// This catches errors on idle clients in the pool. Without this
// handler an unexpected disconnect would crash the process.
logger.error(
console.error(
`Unexpected error on idle ${label} database client: ${err.message}`
);
});
@@ -36,7 +45,7 @@ export function attachPoolErrorHandlers(pool: Pool, label: string): void {
// Set a statement timeout on every new connection so a single slow
// query can't block the pool forever
client.query("SET statement_timeout = '30s'").catch((err: Error) => {
logger.warn(
console.warn(
`Failed to set statement_timeout on ${label} client: ${err.message}`
);
});
@@ -60,4 +69,4 @@ export function createPool(
);
attachPoolErrorHandlers(pool, label);
return pool;
}
}
+64 -15
View File
@@ -2,6 +2,7 @@ import {
pgTable,
serial,
varchar,
unique,
boolean,
integer,
bigint,
@@ -11,7 +12,7 @@ import {
primaryKey,
uniqueIndex
} from "drizzle-orm/pg-core";
import { InferSelectModel } from "drizzle-orm";
import { InferSelectModel, sql } from "drizzle-orm";
import {
domains,
orgs,
@@ -19,12 +20,13 @@ import {
roles,
users,
exitNodes,
sessions,
clients,
resources,
siteResources,
targetHealthCheck,
sites
sites,
clients,
sessions,
labels
} from "./schema";
export const certificates = pgTable("certificates", {
@@ -197,6 +199,42 @@ export const remoteExitNodes = pgTable("remoteExitNode", {
})
});
export const remoteExitNodeResources = pgTable("remoteExitNodeResources", {
remoteExitNodeResourceId: serial("remoteExitNodeResourceId").primaryKey(),
remoteExitNodeId: varchar("remoteExitNodeId")
.notNull()
.references(() => remoteExitNodes.remoteExitNodeId, {
onDelete: "cascade"
}),
destination: varchar("destination").notNull() // a cidr range
});
export const remoteExitNodePreferenceLabels = pgTable(
// this controls what sites are enforced to connect to this node
"remoteExitNodePreferenceLabels",
{
remoteExitNodePreferenceLabelId: serial(
"remoteExitNodePreferenceLabelId"
).primaryKey(),
remoteExitNodeId: varchar("remoteExitNodeId")
.references(() => remoteExitNodes.remoteExitNodeId, {
onDelete: "cascade"
})
.notNull(),
labelId: integer("labelId")
.references(() => labels.labelId, {
onDelete: "cascade"
})
.notNull()
},
(t) => [
unique("remote_exit_node_preference_label_uniq").on(
t.remoteExitNodeId,
t.labelId
)
]
);
export const remoteExitNodeSessions = pgTable("remoteExitNodeSession", {
sessionId: varchar("id").primaryKey(),
remoteExitNodeId: varchar("remoteExitNodeId")
@@ -207,17 +245,28 @@ export const remoteExitNodeSessions = pgTable("remoteExitNodeSession", {
expiresAt: bigint("expiresAt", { mode: "number" }).notNull()
});
export const loginPage = pgTable("loginPage", {
loginPageId: serial("loginPageId").primaryKey(),
subdomain: varchar("subdomain"),
fullDomain: varchar("fullDomain"),
exitNodeId: integer("exitNodeId").references(() => exitNodes.exitNodeId, {
onDelete: "set null"
}),
domainId: varchar("domainId").references(() => domains.domainId, {
onDelete: "set null"
})
});
export const loginPage = pgTable(
"loginPage",
{
loginPageId: serial("loginPageId").primaryKey(),
subdomain: varchar("subdomain"),
fullDomain: varchar("fullDomain"),
exitNodeId: integer("exitNodeId").references(
() => exitNodes.exitNodeId,
{
onDelete: "set null"
}
),
domainId: varchar("domainId").references(() => domains.domainId, {
onDelete: "set null"
})
},
(t) => [
index("idx_loginpage_fulldomain")
.on(t.fullDomain)
.where(sql`${t.fullDomain} IS NOT NULL`)
]
);
export const loginPageOrg = pgTable("loginPageOrg", {
loginPageId: integer("loginPageId")
File diff suppressed because it is too large Load Diff
+278 -38
View File
@@ -17,22 +17,37 @@ import {
resourceHeaderAuth,
ResourceHeaderAuth,
resourceRules,
resourcePolicyRules,
resources,
roleResources,
rolePolicies,
sessions,
userResources,
userPolicies,
users,
ResourceHeaderAuthExtendedCompatibility,
resourceHeaderAuthExtendedCompatibility
resourceHeaderAuthExtendedCompatibility,
resourcePolicies,
resourcePolicyPincode,
ResourcePolicyPincode,
resourcePolicyPassword,
ResourcePolicyPassword,
resourcePolicyHeaderAuth,
ResourcePolicyHeaderAuth
} from "@server/db";
import { and, eq, inArray, or, sql } from "drizzle-orm";
import { alias } from "@server/db";
import { and, eq, inArray, isNull, or, sql } from "drizzle-orm";
import logger from "@server/logger";
export type ResourceWithAuth = {
resource: Resource | null;
pincode: ResourcePincode | null;
password: ResourcePassword | null;
headerAuth: ResourceHeaderAuth | null;
pincode: ResourcePincode | ResourcePolicyPincode | null;
password: ResourcePassword | ResourcePolicyPassword | null;
headerAuth: ResourceHeaderAuth | ResourcePolicyHeaderAuth | null;
headerAuthExtendedCompatibility: ResourceHeaderAuthExtendedCompatibility | null;
applyRules: boolean | null;
sso: boolean | null;
emailWhitelistEnabled: boolean | null;
org: Org;
};
@@ -57,6 +72,33 @@ export async function getResourceByDomain(
wildcardCandidates.push(`*.${parts.slice(i).join(".")}`);
}
const sharedPolicy = alias(resourcePolicies, "sharedPolicy");
const defaultPolicy = alias(resourcePolicies, "defaultPolicy");
const sharedPolicyPincode = alias(
resourcePolicyPincode,
"sharedPolicyPincode"
);
const defaultPolicyPincode = alias(
resourcePolicyPincode,
"defaultPolicyPincode"
);
const sharedPolicyPassword = alias(
resourcePolicyPassword,
"sharedPolicyPassword"
);
const defaultPolicyPassword = alias(
resourcePolicyPassword,
"defaultPolicyPassword"
);
const sharedPolicyHeaderAuth = alias(
resourcePolicyHeaderAuth,
"sharedPolicyHeaderAuth"
);
const defaultPolicyHeaderAuth = alias(
resourcePolicyHeaderAuth,
"defaultPolicyHeaderAuth"
);
const potentialResults = await db
.select()
.from(resources)
@@ -79,6 +121,59 @@ export async function getResourceByDomain(
resources.resourceId
)
)
.leftJoin(
sharedPolicy,
eq(sharedPolicy.resourcePolicyId, resources.resourcePolicyId)
)
.leftJoin(
sharedPolicyPincode,
eq(
sharedPolicyPincode.resourcePolicyId,
sharedPolicy.resourcePolicyId
)
)
.leftJoin(
sharedPolicyPassword,
eq(
sharedPolicyPassword.resourcePolicyId,
sharedPolicy.resourcePolicyId
)
)
.leftJoin(
sharedPolicyHeaderAuth,
eq(
sharedPolicyHeaderAuth.resourcePolicyId,
sharedPolicy.resourcePolicyId
)
)
.leftJoin(
defaultPolicy,
eq(
defaultPolicy.resourcePolicyId,
resources.defaultResourcePolicyId
)
)
.leftJoin(
defaultPolicyPincode,
eq(
defaultPolicyPincode.resourcePolicyId,
defaultPolicy.resourcePolicyId
)
)
.leftJoin(
defaultPolicyPassword,
eq(
defaultPolicyPassword.resourcePolicyId,
defaultPolicy.resourcePolicyId
)
)
.leftJoin(
defaultPolicyHeaderAuth,
eq(
defaultPolicyHeaderAuth.resourcePolicyId,
defaultPolicy.resourcePolicyId
)
)
.innerJoin(orgs, eq(orgs.orgId, resources.orgId))
.where(
or(
@@ -108,13 +203,51 @@ export async function getResourceByDomain(
return null;
}
// If a shared (custom) policy is assigned to the resource, use ONLY
// its values — do not fall back to the default policy. The default
// policy is only consulted when no shared policy is assigned at all.
const hasSharedPolicy = result.sharedPolicy !== null;
const effectivePolicyPincode = hasSharedPolicy
? result.sharedPolicyPincode
: (result.defaultPolicyPincode ?? null);
const effectivePolicyPassword = hasSharedPolicy
? result.sharedPolicyPassword
: (result.defaultPolicyPassword ?? null);
const effectivePolicyHeaderAuth = hasSharedPolicy
? result.sharedPolicyHeaderAuth
: (result.defaultPolicyHeaderAuth ?? null);
const selectedPolicy = hasSharedPolicy
? result.sharedPolicy
: result.defaultPolicy;
const effectiveApplyRules =
selectedPolicy?.applyRules ?? result.resources.applyRules;
const effectiveSSO = selectedPolicy?.sso ?? result.resources.sso;
const effectiveEmailWhitelistEnabled =
selectedPolicy?.emailWhitelistEnabled ??
result.resources.emailWhitelistEnabled;
return {
resource: result.resources,
pincode: result.resourcePincode,
password: result.resourcePassword,
headerAuth: result.resourceHeaderAuth,
headerAuthExtendedCompatibility:
result.resourceHeaderAuthExtendedCompatibility,
resource: {
...result.resources,
applyRules: effectiveApplyRules,
sso: effectiveSSO,
emailWhitelistEnabled: effectiveEmailWhitelistEnabled
}, // doing this for backward compatability so the remote nodes get the value as part of the resource struct
pincode: effectivePolicyPincode ?? result.resourcePincode,
password: effectivePolicyPassword ?? result.resourcePassword,
headerAuth: effectivePolicyHeaderAuth ?? result.resourceHeaderAuth,
headerAuthExtendedCompatibility: effectivePolicyHeaderAuth
? ({
headerAuthExtendedCompatibilityId: 0,
resourceId: result.resources.resourceId,
extendedCompatibilityIsActivated:
effectivePolicyHeaderAuth.extendedCompatibility
} as ResourceHeaderAuthExtendedCompatibility)
: result.resourceHeaderAuthExtendedCompatibility,
applyRules: effectiveApplyRules,
sso: effectiveSSO,
emailWhitelistEnabled: effectiveEmailWhitelistEnabled,
org: result.orgs
};
}
@@ -154,58 +287,165 @@ export async function getRoleName(roleId: number): Promise<string | null> {
}
/**
* Check if role has access to resource
* Check if role has access to resource (direct or via resource policy)
*/
export async function getRoleResourceAccess(
resourceId: number,
roleIds: number[]
) {
const roleResourceAccess = await db
.select()
.from(roleResources)
.where(
and(
eq(roleResources.resourceId, resourceId),
inArray(roleResources.roleId, roleIds)
const [direct, viaPolicies] = await Promise.all([
db
.select()
.from(roleResources)
.where(
and(
eq(roleResources.resourceId, resourceId),
inArray(roleResources.roleId, roleIds)
)
),
db
.select({
roleId: rolePolicies.roleId,
resourcePolicyId: rolePolicies.resourcePolicyId
})
.from(rolePolicies)
.innerJoin(
resources,
// Shared policy wins; only use default policy when no shared
// policy is assigned to the resource.
or(
eq(
resources.resourcePolicyId,
rolePolicies.resourcePolicyId
),
and(
isNull(resources.resourcePolicyId),
eq(
resources.defaultResourcePolicyId,
rolePolicies.resourcePolicyId
)
)
)
)
);
.where(
and(
eq(resources.resourceId, resourceId),
inArray(rolePolicies.roleId, roleIds)
)
)
]);
return roleResourceAccess.length > 0 ? roleResourceAccess : null;
const combined = [...direct, ...viaPolicies];
return combined.length > 0 ? combined : null;
}
/**
* Check if user has direct access to resource
* Check if user has access to resource (direct or via resource policy)
*/
export async function getUserResourceAccess(
userId: string,
resourceId: number
) {
const userResourceAccess = await db
.select()
.from(userResources)
.where(
and(
eq(userResources.userId, userId),
eq(userResources.resourceId, resourceId)
const [direct, viaPolicies] = await Promise.all([
db
.select()
.from(userResources)
.where(
and(
eq(userResources.userId, userId),
eq(userResources.resourceId, resourceId)
)
)
)
.limit(1);
.limit(1),
db
.select({
userId: userPolicies.userId,
resourcePolicyId: userPolicies.resourcePolicyId
})
.from(userPolicies)
.innerJoin(
resources,
// Shared policy wins; only use default policy when no shared
// policy is assigned to the resource.
or(
eq(
resources.resourcePolicyId,
userPolicies.resourcePolicyId
),
and(
isNull(resources.resourcePolicyId),
eq(
resources.defaultResourcePolicyId,
userPolicies.resourcePolicyId
)
)
)
)
.where(
and(
eq(resources.resourceId, resourceId),
eq(userPolicies.userId, userId)
)
)
.limit(1)
]);
return userResourceAccess.length > 0 ? userResourceAccess[0] : null;
return direct[0] ?? viaPolicies[0] ?? null;
}
/**
* Get resource rules for a given resource
* Get resource rules for a given resource (direct and via resource policy)
*/
export async function getResourceRules(
resourceId: number
): Promise<ResourceRule[]> {
const rules = await db
.select()
.from(resourceRules)
.where(eq(resourceRules.resourceId, resourceId));
const [directRules, policyRules] = await Promise.all([
db
.select()
.from(resourceRules)
.where(eq(resourceRules.resourceId, resourceId)),
db
.select({
ruleId: resourcePolicyRules.ruleId,
resourceId: sql<number>`${resourceId}`,
enabled: resourcePolicyRules.enabled,
priority: resourcePolicyRules.priority,
action: resourcePolicyRules.action,
match: resourcePolicyRules.match,
value: resourcePolicyRules.value
})
.from(resourcePolicyRules)
.innerJoin(
resources,
// Shared policy wins; only use default policy when no shared
// policy is assigned to the resource.
or(
eq(
resources.resourcePolicyId,
resourcePolicyRules.resourcePolicyId
),
and(
isNull(resources.resourcePolicyId),
eq(
resources.defaultResourcePolicyId,
resourcePolicyRules.resourcePolicyId
)
)
)
)
.where(eq(resources.resourceId, resourceId))
]);
return rules;
const maxDirectPriority = directRules.reduce(
(max, r) => Math.max(max, r.priority),
0
);
const offsetPolicyRules = policyRules.map((r) => ({
...r,
priority: maxDirectPriority + r.priority
}));
return [...directRules, ...offsetPolicyRules] as ResourceRule[];
}
/**
+12 -50
View File
@@ -1,6 +1,5 @@
import { drizzle as DrizzleSqlite } from "drizzle-orm/better-sqlite3";
import Database from "better-sqlite3";
import type BetterSqlite3 from "better-sqlite3";
import * as schema from "./schema/schema";
import path from "path";
import fs from "fs";
@@ -12,68 +11,31 @@ export const exists = checkFileExists(location);
bootstrapVolume();
/**
* Wraps better-sqlite3 Statement to call `finalize()` immediately after
* execution, freeing native sqlite3_stmt memory deterministically instead
* of waiting for GC. Fixes steady off-heap growth under load (#2120).
* WARNING: Finalizes after first execution — incompatible with drizzle's
* reusable .prepare() builders. No such usage exists in this codebase.
*/
function autoFinalizeStatement(
stmt: BetterSqlite3.Statement
): BetterSqlite3.Statement {
const wrapExec = <T extends (...args: any[]) => any>(fn: T): T => {
return function (this: any, ...args: any[]) {
try {
return fn.apply(this, args);
} finally {
try {
// finalize() exists on the native Statement at runtime but
// is missing from @types/better-sqlite3.
(stmt as any).finalize();
} catch {
// Already finalized — harmless
}
}
} as unknown as T;
};
stmt.run = wrapExec(stmt.run);
stmt.get = wrapExec(stmt.get);
stmt.all = wrapExec(stmt.all);
return stmt;
}
function createDb() {
const sqlite = new Database(location);
if (process.env.ENABLE_SQLITE_WAL_MODE == "true") {
// Enable WAL mode — allows concurrent readers + single writer, preventing
// contention across subsystems (verifySession, Traefik, audit, ping).
// NOTE: journal_mode persists in the DB file once set; unsetting this
// env var does NOT revert an existing WAL database.
sqlite.pragma("journal_mode = WAL");
// NORMAL sync mode: safe with WAL, reduces write lock hold time.
sqlite.pragma("synchronous = NORMAL");
}
// Wait up to 5s on SQLITE_BUSY instead of failing — prevents audit log
// retry loops that accumulate memory.
sqlite.pragma("busy_timeout = 5000");
// No busy_timeout pragma: better-sqlite3 already arms
// sqlite3_busy_timeout(db, 5000) via its default `timeout` option
// (lib/database.js), so an explicit pragma is redundant.
// 64 MB page cache (default 2 MB) — reduces I/O round-trips on large
// TraefikConfigManager JOINs that block the event loop.
sqlite.pragma("cache_size = -65536");
// Intentionally NOT setting cache_size or mmap_size: a large page cache plus
// a multi-hundred-MB mmap region inflate RSS and cause page-cache thrashing
// on small (~1 GB) instances. Leave SQLite on its conservative defaults.
// 256 MB memory-mapped I/O — OS serves reads from page cache directly,
// reducing event-loop blocking.
sqlite.pragma("mmap_size = 268435456");
// Wrap prepare() so every drizzle-orm statement is auto-finalized after
// first use, preventing sqlite3_stmt accumulation between GC cycles.
const originalPrepare = sqlite.prepare.bind(sqlite);
(sqlite as any).prepare = function autoFinalizePrepare(source: string) {
return autoFinalizeStatement(originalPrepare(source));
};
// Intentionally NOT wrapping prepare()/statements: better-sqlite3 finalizes
// sqlite3_stmt in the Statement destructor at GC, and drizzle-orm prepares a
// fresh statement per query (no statement cache), so statements cannot
// accumulate. better-sqlite3 11.x exposes no Statement.finalize() at all.
return DrizzleSqlite(sqlite, {
schema
+1
View File
@@ -4,3 +4,4 @@ export * from "./safeRead";
export * from "./schema/schema";
export * from "./schema/privateSchema";
export * from "./migrate";
export { alias } from "drizzle-orm/sqlite-core";
+39 -3
View File
@@ -12,6 +12,7 @@ import {
clients,
domains,
exitNodes,
labels,
orgs,
resources,
roles,
@@ -21,9 +22,6 @@ import {
targetHealthCheck,
users
} from "./schema";
import { serial, varchar } from "drizzle-orm/mysql-core";
import { pgTable } from "drizzle-orm/pg-core";
import { bigint } from "zod";
export const certificates = sqliteTable("certificates", {
certId: integer("certId").primaryKey({ autoIncrement: true }),
@@ -195,6 +193,44 @@ export const remoteExitNodes = sqliteTable("remoteExitNode", {
})
});
export const remoteExitNodeResources = sqliteTable("remoteExitNodeResources", {
remoteExitNodeResourceId: integer("remoteExitNodeResourceId").primaryKey({
autoIncrement: true
}),
remoteExitNodeId: text("remoteExitNodeId")
.notNull()
.references(() => remoteExitNodes.remoteExitNodeId, {
onDelete: "cascade"
}),
destination: text("destination").notNull() // a cidr range
});
export const remoteExitNodePreferenceLabels = sqliteTable(
// this controls what sites are enforced to connect to this node
"remoteExitNodePreferenceLabels",
{
remoteExitNodePreferenceLabelId: integer(
"remoteExitNodePreferenceLabelId"
).primaryKey({ autoIncrement: true }),
remoteExitNodeId: text("remoteExitNodeId")
.references(() => remoteExitNodes.remoteExitNodeId, {
onDelete: "cascade"
})
.notNull(),
labelId: integer("labelId")
.references(() => labels.labelId, {
onDelete: "cascade"
})
.notNull()
},
(t) => [
uniqueIndex("remote_exit_node_preference_label_uniq").on(
t.remoteExitNodeId,
t.labelId
)
]
);
export const remoteExitNodeSessions = sqliteTable("remoteExitNodeSession", {
sessionId: text("id").primaryKey(),
remoteExitNodeId: text("remoteExitNodeId")
+217 -17
View File
@@ -20,8 +20,10 @@ export const domains = sqliteTable("domains", {
failed: integer("failed", { mode: "boolean" }).notNull().default(false),
tries: integer("tries").notNull().default(0),
certResolver: text("certResolver"),
customCertResolver: text("customCertResolver"),
preferWildcardCert: integer("preferWildcardCert", { mode: "boolean" }),
errorMessage: text("errorMessage")
errorMessage: text("errorMessage"),
lastCheckedAt: integer("lastCheckedAt")
});
export const dnsRecords = sqliteTable("dnsRecords", {
@@ -62,7 +64,13 @@ export const orgs = sqliteTable("orgs", {
sshCaPrivateKey: text("sshCaPrivateKey"), // Encrypted SSH CA private key (PEM format)
sshCaPublicKey: text("sshCaPublicKey"), // SSH CA public key (OpenSSH format)
isBillingOrg: integer("isBillingOrg", { mode: "boolean" }),
billingOrgId: text("billingOrgId")
billingOrgId: text("billingOrgId"),
settingsEnableGlobalNewtAutoUpdate: integer(
"settingsEnableGlobalNewtAutoUpdate",
{ mode: "boolean" }
)
.notNull()
.default(false)
});
export const userDomains = sqliteTable("userDomains", {
@@ -116,11 +124,29 @@ export const sites = sqliteTable("sites", {
dockerSocketEnabled: integer("dockerSocketEnabled", { mode: "boolean" })
.notNull()
.default(true),
autoUpdateEnabled: integer("autoUpdateEnabled", { mode: "boolean" })
.notNull()
.default(false),
autoUpdateOverrideOrg: integer("autoUpdateOverrideOrg", {
mode: "boolean"
})
.notNull()
.default(false),
status: text("status").$type<"pending" | "approved">().default("approved")
});
export const resources = sqliteTable("resources", {
resourceId: integer("resourceId").primaryKey({ autoIncrement: true }),
resourcePolicyId: integer("resourcePolicyId").references(
() => resourcePolicies.resourcePolicyId,
{ onDelete: "set null" }
),
defaultResourcePolicyId: integer("defaultResourcePolicyId").references(
() => resourcePolicies.resourcePolicyId,
{
onDelete: "restrict"
}
),
resourceGuid: text("resourceGuid", { length: 36 })
.unique()
.notNull()
@@ -141,16 +167,12 @@ export const resources = sqliteTable("resources", {
blockAccess: integer("blockAccess", { mode: "boolean" })
.notNull()
.default(false),
sso: integer("sso", { mode: "boolean" }).notNull().default(true),
http: integer("http", { mode: "boolean" }).notNull().default(true),
protocol: text("protocol").notNull(),
proxyPort: integer("proxyPort"),
emailWhitelistEnabled: integer("emailWhitelistEnabled", { mode: "boolean" })
.notNull()
.default(false),
applyRules: integer("applyRules", { mode: "boolean" })
.notNull()
.default(false),
sso: integer("sso", { mode: "boolean" }),
emailWhitelistEnabled: integer("emailWhitelistEnabled", {
mode: "boolean"
}),
applyRules: integer("applyRules", { mode: "boolean" }),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
stickySession: integer("stickySession", { mode: "boolean" })
.notNull()
@@ -167,7 +189,6 @@ export const resources = sqliteTable("resources", {
.notNull()
.default(false),
proxyProtocolVersion: integer("proxyProtocolVersion").default(1),
maintenanceModeEnabled: integer("maintenanceModeEnabled", {
mode: "boolean"
})
@@ -181,7 +202,15 @@ export const resources = sqliteTable("resources", {
maintenanceEstimatedTime: text("maintenanceEstimatedTime"),
postAuthPath: text("postAuthPath"),
health: text("health").default("unknown"), // "healthy", "unhealthy", "unknown"
wildcard: integer("wildcard", { mode: "boolean" }).notNull().default(false)
wildcard: integer("wildcard", { mode: "boolean" }).notNull().default(false),
mode: text("mode").default("http").notNull(), // rdp, ssh, http, vnc
pamMode: text("pamMode")
.$type<"passthrough" | "push">()
.default("passthrough"),
authDaemonMode: text("authDaemonMode")
.$type<"site" | "remote" | "native">()
.default("site"),
authDaemonPort: integer("authDaemonPort").default(22123)
});
export const labels = sqliteTable("labels", {
@@ -195,6 +224,23 @@ export const labels = sqliteTable("labels", {
.notNull()
});
export const launcherViews = sqliteTable("launcherViews", {
viewId: integer("viewId").primaryKey({ autoIncrement: true }),
orgId: text("orgId")
.notNull()
.references(() => orgs.orgId, { onDelete: "cascade" }),
userId: text("userId").references(() => users.userId, {
onDelete: "cascade"
}),
name: text("name").notNull(),
config: text("config").notNull(),
isDefault: integer("isDefault", { mode: "boolean" })
.notNull()
.default(false),
createdAt: text("createdAt").notNull(),
updatedAt: text("updatedAt").notNull()
});
export const siteLabels = sqliteTable(
"siteLabels",
{
@@ -294,7 +340,12 @@ export const targets = sqliteTable("targets", {
pathMatchType: text("pathMatchType"), // exact, prefix, regex
rewritePath: text("rewritePath"), // if set, rewrites the path to this value before sending to the target
rewritePathType: text("rewritePathType"), // exact, prefix, regex, stripPrefix
priority: integer("priority").notNull().default(100)
priority: integer("priority").notNull().default(100),
mode: text("mode")
.$type<"http" | "tcp" | "udp" | "ssh" | "rdp" | "vnc">()
.notNull()
.default("http"),
authToken: text("authToken")
});
export const targetHealthCheck = sqliteTable("targetHealthCheck", {
@@ -373,11 +424,11 @@ export const siteResources = sqliteTable("siteResources", {
niceId: text("niceId").notNull(),
name: text("name").notNull(),
ssl: integer("ssl", { mode: "boolean" }).notNull().default(false),
mode: text("mode").$type<"host" | "cidr" | "http">().notNull(), // "host" | "cidr" | "http"
mode: text("mode").$type<"host" | "cidr" | "http" | "ssh">().notNull(), // "host" | "cidr" | "http"
scheme: text("scheme").$type<"http" | "https">(), // only for when we are doing https or http mode
proxyPort: integer("proxyPort"), // only for port mode
destinationPort: integer("destinationPort"), // only for port mode
destination: text("destination").notNull(), // ip, cidr, hostname
destination: text("destination"), // ip, cidr, hostname
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
alias: text("alias"),
aliasAddress: text("aliasAddress"),
@@ -387,8 +438,11 @@ export const siteResources = sqliteTable("siteResources", {
.notNull()
.default(false),
authDaemonPort: integer("authDaemonPort").default(22123),
pamMode: text("pamMode")
.$type<"passthrough" | "push">()
.default("passthrough"),
authDaemonMode: text("authDaemonMode")
.$type<"site" | "remote">()
.$type<"site" | "remote" | "native">()
.default("site"),
domainId: text("domainId").references(() => domains.domainId, {
onDelete: "set null"
@@ -1001,6 +1055,47 @@ export const resourceHeaderAuth = sqliteTable("resourceHeaderAuth", {
headerAuthHash: text("headerAuthHash").notNull()
});
export const resourcePolicyPincode = sqliteTable("resourcePolicyPincode", {
pincodeId: integer("pincodeId").primaryKey({ autoIncrement: true }),
pincodeHash: text("pincodeHash").notNull(),
digitLength: integer("digitLength").notNull(),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
})
});
export const resourcePolicyPassword = sqliteTable("resourcePolicyPassword", {
passwordId: integer("passwordId").primaryKey({ autoIncrement: true }),
passwordHash: text("passwordHash").notNull(),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
})
});
export const resourcePolicyHeaderAuth = sqliteTable(
"resourcePolicyHeaderAuth",
{
headerAuthId: integer("headerAuthId").primaryKey({
autoIncrement: true
}),
headerAuthHash: text("headerAuthHash").notNull(),
extendedCompatibility: integer("extendedCompatibility", {
mode: "boolean"
})
.notNull()
.default(true),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
})
}
);
export const resourceHeaderAuthExtendedCompatibility = sqliteTable(
"resourceHeaderAuthExtendedCompatibility",
{
@@ -1029,6 +1124,7 @@ export const resourceAccessToken = sqliteTable("resourceAccessToken", {
resourceId: integer("resourceId")
.notNull()
.references(() => resources.resourceId, { onDelete: "cascade" }),
path: text("path"),
tokenHash: text("tokenHash").notNull(),
sessionLength: integer("sessionLength").notNull(),
expiresAt: integer("expiresAt"),
@@ -1075,6 +1171,24 @@ export const resourceSessions = sqliteTable("resourceSessions", {
onDelete: "cascade"
}
),
policyPasswordId: integer("policyPasswordId").references(
() => resourcePolicyPassword.passwordId,
{
onDelete: "cascade"
}
),
policyPincodeId: integer("policyPincodeId").references(
() => resourcePolicyPincode.pincodeId,
{
onDelete: "cascade"
}
),
policyWhitelistId: integer("policyWhitelistId").references(
() => resourcePolicyWhiteList.whitelistId,
{
onDelete: "cascade"
}
),
issuedAt: integer("issuedAt")
});
@@ -1115,6 +1229,79 @@ export const resourceRules = sqliteTable("resourceRules", {
value: text("value").notNull()
});
export const rolePolicies = sqliteTable("rolePolicies", {
roleId: integer("roleId")
.notNull()
.references(() => roles.roleId, { onDelete: "cascade" }),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
})
});
export const userPolicies = sqliteTable("userPolicies", {
userId: text("userId")
.notNull()
.references(() => users.userId, { onDelete: "cascade" }),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
})
});
export const resourcePolicyWhiteList = sqliteTable("resourcePolicyWhitelist", {
whitelistId: integer("id").primaryKey({ autoIncrement: true }),
email: text("email").notNull(),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
})
});
export const resourcePolicyRules = sqliteTable("resourcePolicyRules", {
ruleId: integer("ruleId").primaryKey({ autoIncrement: true }),
resourcePolicyId: integer("resourcePolicyId")
.notNull()
.references(() => resourcePolicies.resourcePolicyId, {
onDelete: "cascade"
}),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
priority: integer("priority").notNull(),
action: text("action").$type<"ACCEPT" | "DROP" | "PASS">().notNull(),
match: text("match")
.$type<"CIDR" | "PATH" | "IP" | "COUNTRY" | "ASN" | "REGION">()
.notNull(),
value: text("value").notNull()
});
export const resourcePolicies = sqliteTable("resourcePolicies", {
resourcePolicyId: integer("resourcePolicyId").primaryKey(),
sso: integer("sso", { mode: "boolean" }).notNull().default(true),
applyRules: integer("applyRules", { mode: "boolean" })
.notNull()
.default(false),
scope: text("scope")
.$type<"global" | "resource">()
.notNull()
.default("global"),
emailWhitelistEnabled: integer("emailWhitelistEnabled", { mode: "boolean" })
.notNull()
.default(false),
niceId: text("niceId").notNull(),
idpId: integer("idpId").references(() => idp.idpId, {
onDelete: "set null"
}),
name: text("name").notNull(),
orgId: text("orgId")
.references(() => orgs.orgId, {
onDelete: "cascade"
})
.notNull()
});
export const supporterKey = sqliteTable("supporterKey", {
keyId: integer("keyId").primaryKey({ autoIncrement: true }),
key: text("key").notNull(),
@@ -1382,3 +1569,16 @@ export type RoundTripMessageTracker = InferSelectModel<
>;
export type StatusHistory = InferSelectModel<typeof statusHistory>;
export type Label = InferSelectModel<typeof labels>;
export type LauncherView = InferSelectModel<typeof launcherViews>;
export type ResourcePolicy = InferSelectModel<typeof resourcePolicies>;
export type ResourcePolicyPincode = InferSelectModel<
typeof resourcePolicyPincode
>;
export type ResourcePolicyPassword = InferSelectModel<
typeof resourcePolicyPassword
>;
export type ResourcePolicyHeaderAuth = InferSelectModel<
typeof resourcePolicyHeaderAuth
>;
export type RolePolicy = InferSelectModel<typeof rolePolicies>;
export type UserPolicy = InferSelectModel<typeof userPolicies>;
+3 -1
View File
@@ -1,5 +1,5 @@
#! /usr/bin/env node
import "./extendZod.ts";
import "./extendZod";
import { runSetupFunctions } from "./setup";
import { createApiServer } from "./apiServer";
@@ -24,6 +24,7 @@ import license from "#dynamic/license/license";
import { initLogCleanupInterval } from "@server/lib/cleanupLogs";
import { initAcmeCertSync } from "#dynamic/lib/acmeCertSync";
import { fetchServerIp } from "@server/lib/serverIpService";
import { startRebuildQueueProcessor } from "@server/lib/rebuildClientAssociations";
async function startServers() {
await setHostMeta();
@@ -41,6 +42,7 @@ async function startServers() {
initLogCleanupInterval();
initAcmeCertSync();
startRebuildQueueProcessor();
// Start all servers
const apiServer = createApiServer();
+11 -3
View File
@@ -152,11 +152,19 @@ function getOpenApiDocumentation() {
if (!hasExistingResponses) {
def.route.responses = {
"*": {
description: "",
"200": {
description: "Successful response",
content: {
"application/json": {
schema: z.object({})
schema: z.object({
data: z
.record(z.string(), z.any())
.nullable(),
success: z.boolean(),
error: z.boolean(),
message: z.string(),
status: z.number()
})
}
}
}
+11 -3
View File
@@ -221,10 +221,18 @@ async function handleResource(
)
.where(eq(targets.resourceId, resource.resourceId));
const monitoredTargets = otherTargets.filter(
(t) => t.hcHealth !== "unknown"
);
let health = "healthy";
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
const allUnknown = monitoredTargets.length === 0;
const allHealthy = monitoredTargets.every(
(t) => t.hcHealth === "healthy"
);
const allUnhealthy = monitoredTargets.every(
(t) => t.hcHealth === "unhealthy"
);
if (allUnknown) {
logger.debug(
+36 -26
View File
@@ -1,28 +1,39 @@
export enum FeatureId {
export enum LimitId {
USERS = "users",
SITES = "sites",
EGRESS_DATA_MB = "egressDataMb",
DOMAINS = "domains",
REMOTE_EXIT_NODES = "remoteExitNodes",
ORGINIZATIONS = "organizations",
ORGANIZATIONS = "organizations",
PUBLIC_RESOURCES = "publicResources",
PRIVATE_RESOURCES = "privateResources",
MACHINE_CLIENTS = "machineClients",
TIER1 = "tier1"
}
export async function getFeatureDisplayName(featureId: FeatureId): Promise<string> {
export async function getFeatureDisplayName(
featureId: LimitId
): Promise<string> {
switch (featureId) {
case FeatureId.USERS:
case LimitId.USERS:
return "Users";
case FeatureId.SITES:
case LimitId.SITES:
return "Sites";
case FeatureId.EGRESS_DATA_MB:
case LimitId.EGRESS_DATA_MB:
return "Egress Data (MB)";
case FeatureId.DOMAINS:
case LimitId.DOMAINS:
return "Domains";
case FeatureId.REMOTE_EXIT_NODES:
case LimitId.REMOTE_EXIT_NODES:
return "Remote Exit Nodes";
case FeatureId.ORGINIZATIONS:
case LimitId.ORGANIZATIONS:
return "Organizations";
case FeatureId.TIER1:
case LimitId.PUBLIC_RESOURCES:
return "Public Resources";
case LimitId.PRIVATE_RESOURCES:
return "Private Resources";
case LimitId.MACHINE_CLIENTS:
return "Machine Clients";
case LimitId.TIER1:
return "Home Lab";
default:
return featureId;
@@ -30,15 +41,16 @@ export async function getFeatureDisplayName(featureId: FeatureId): Promise<strin
}
// this is from the old system
export const FeatureMeterIds: Partial<Record<FeatureId, string>> = { // right now we are not charging for any data
export const FeatureMeterIds: Partial<Record<LimitId, string>> = {
// right now we are not charging for any data
// [FeatureId.EGRESS_DATA_MB]: "mtr_61Srreh9eWrExDSCe41D3Ee2Ir7Wm5YW"
};
export const FeatureMeterIdsSandbox: Partial<Record<FeatureId, string>> = {
export const FeatureMeterIdsSandbox: Partial<Record<LimitId, string>> = {
// [FeatureId.EGRESS_DATA_MB]: "mtr_test_61Snh2a2m6qome5Kv41DCpkOb237B3dQ"
};
export function getFeatureMeterId(featureId: FeatureId): string | undefined {
export function getFeatureMeterId(featureId: LimitId): string | undefined {
if (
process.env.ENVIRONMENT == "prod" &&
process.env.SANDBOX_MODE !== "true"
@@ -49,22 +61,20 @@ export function getFeatureMeterId(featureId: FeatureId): string | undefined {
}
}
export function getFeatureIdByMetricId(
metricId: string
): FeatureId | undefined {
return (Object.entries(FeatureMeterIds) as [FeatureId, string][]).find(
export function getFeatureIdByMetricId(metricId: string): LimitId | undefined {
return (Object.entries(FeatureMeterIds) as [LimitId, string][]).find(
([_, v]) => v === metricId
)?.[0];
}
export type FeaturePriceSet = Partial<Record<FeatureId, string>>;
export type FeaturePriceSet = Partial<Record<LimitId, string>>;
export const tier1FeaturePriceSet: FeaturePriceSet = {
[FeatureId.TIER1]: "price_1SzVE3D3Ee2Ir7Wm6wT5Dl3G"
[LimitId.TIER1]: "price_1SzVE3D3Ee2Ir7Wm6wT5Dl3G"
};
export const tier1FeaturePriceSetSandbox: FeaturePriceSet = {
[FeatureId.TIER1]: "price_1SxgpPDCpkOb237Bfo4rIsoT"
[LimitId.TIER1]: "price_1SxgpPDCpkOb237Bfo4rIsoT"
};
export function getTier1FeaturePriceSet(): FeaturePriceSet {
@@ -79,11 +89,11 @@ export function getTier1FeaturePriceSet(): FeaturePriceSet {
}
export const tier2FeaturePriceSet: FeaturePriceSet = {
[FeatureId.USERS]: "price_1SzVCcD3Ee2Ir7Wmn6U3KvPN"
[LimitId.USERS]: "price_1SzVCcD3Ee2Ir7Wmn6U3KvPN"
};
export const tier2FeaturePriceSetSandbox: FeaturePriceSet = {
[FeatureId.USERS]: "price_1SxaEHDCpkOb237BD9lBkPiR"
[LimitId.USERS]: "price_1SxaEHDCpkOb237BD9lBkPiR"
};
export function getTier2FeaturePriceSet(): FeaturePriceSet {
@@ -98,11 +108,11 @@ export function getTier2FeaturePriceSet(): FeaturePriceSet {
}
export const tier3FeaturePriceSet: FeaturePriceSet = {
[FeatureId.USERS]: "price_1SzVDKD3Ee2Ir7WmPtOKNusv"
[LimitId.USERS]: "price_1SzVDKD3Ee2Ir7WmPtOKNusv"
};
export const tier3FeaturePriceSetSandbox: FeaturePriceSet = {
[FeatureId.USERS]: "price_1SxaEODCpkOb237BiXdCBSfs"
[LimitId.USERS]: "price_1SxaEODCpkOb237BiXdCBSfs"
};
export function getTier3FeaturePriceSet(): FeaturePriceSet {
@@ -116,7 +126,7 @@ export function getTier3FeaturePriceSet(): FeaturePriceSet {
}
}
export function getFeatureIdByPriceId(priceId: string): FeatureId | undefined {
export function getFeatureIdByPriceId(priceId: string): LimitId | undefined {
// Check all feature price sets
const allPriceSets = [
getTier1FeaturePriceSet(),
@@ -125,7 +135,7 @@ export function getFeatureIdByPriceId(priceId: string): FeatureId | undefined {
];
for (const priceSet of allPriceSets) {
const entry = (Object.entries(priceSet) as [FeatureId, string][]).find(
const entry = (Object.entries(priceSet) as [LimitId, string][]).find(
([_, price]) => price === priceId
);
if (entry) {
+5 -5
View File
@@ -1,19 +1,19 @@
import Stripe from "stripe";
import { FeatureId, FeaturePriceSet } from "./features";
import { LimitId, FeaturePriceSet } from "./features";
import { usageService } from "./usageService";
export async function getLineItems(
featurePriceSet: FeaturePriceSet,
orgId: string,
orgId: string
): Promise<Stripe.Checkout.SessionCreateParams.LineItem[]> {
const users = await usageService.getUsage(orgId, FeatureId.USERS);
const users = await usageService.getUsage(orgId, LimitId.USERS);
return Object.entries(featurePriceSet).map(([featureId, priceId]) => {
let quantity: number | undefined;
if (featureId === FeatureId.USERS) {
if (featureId === LimitId.USERS) {
quantity = users?.instantaneousValue || 1;
} else if (featureId === FeatureId.TIER1) {
} else if (featureId === LimitId.TIER1) {
quantity = 1;
}
+35 -23
View File
@@ -1,70 +1,82 @@
import { FeatureId } from "./features";
import { LimitId } from "./features";
export type LimitSet = Partial<{
[key in FeatureId]: {
[key in LimitId]: {
value: number | null; // null indicates no limit
description?: string;
};
}>;
export const freeLimitSet: LimitSet = {
[FeatureId.SITES]: { value: 5, description: "Basic limit" },
[FeatureId.USERS]: { value: 5, description: "Basic limit" },
[FeatureId.DOMAINS]: { value: 5, description: "Basic limit" },
[FeatureId.REMOTE_EXIT_NODES]: { value: 1, description: "Basic limit" },
[FeatureId.ORGINIZATIONS]: { value: 1, description: "Basic limit" },
[LimitId.SITES]: { value: 5, description: "Basic limit" },
[LimitId.USERS]: { value: 5, description: "Basic limit" },
[LimitId.DOMAINS]: { value: 5, description: "Basic limit" },
[LimitId.REMOTE_EXIT_NODES]: { value: 1, description: "Basic limit" },
[LimitId.ORGANIZATIONS]: { value: 1, description: "Basic limit" },
[LimitId.PUBLIC_RESOURCES]: { value: 15, description: "Basic limit" },
[LimitId.PRIVATE_RESOURCES]: { value: 15, description: "Basic limit" },
[LimitId.MACHINE_CLIENTS]: { value: 5, description: "Basic limit" }
};
export const tier1LimitSet: LimitSet = {
[FeatureId.USERS]: { value: 7, description: "Home limit" },
[FeatureId.SITES]: { value: 10, description: "Home limit" },
[FeatureId.DOMAINS]: { value: 10, description: "Home limit" },
[FeatureId.REMOTE_EXIT_NODES]: { value: 1, description: "Home limit" },
[FeatureId.ORGINIZATIONS]: { value: 1, description: "Home limit" },
[LimitId.USERS]: { value: 7, description: "Home limit" },
[LimitId.SITES]: { value: 10, description: "Home limit" },
[LimitId.DOMAINS]: { value: 10, description: "Home limit" },
[LimitId.REMOTE_EXIT_NODES]: { value: 1, description: "Home limit" },
[LimitId.ORGANIZATIONS]: { value: 1, description: "Home limit" },
[LimitId.PUBLIC_RESOURCES]: { value: 30, description: "Home limit" },
[LimitId.PRIVATE_RESOURCES]: { value: 30, description: "Home limit" },
[LimitId.MACHINE_CLIENTS]: { value: 10, description: "Home limit" }
};
export const tier2LimitSet: LimitSet = {
[FeatureId.USERS]: {
[LimitId.USERS]: {
value: 50,
description: "Team limit"
},
[FeatureId.SITES]: {
[LimitId.SITES]: {
value: 50,
description: "Team limit"
},
[FeatureId.DOMAINS]: {
[LimitId.DOMAINS]: {
value: 50,
description: "Team limit"
},
[FeatureId.REMOTE_EXIT_NODES]: {
[LimitId.REMOTE_EXIT_NODES]: {
value: 3,
description: "Team limit"
},
[FeatureId.ORGINIZATIONS]: {
[LimitId.ORGANIZATIONS]: {
value: 1,
description: "Team limit"
}
},
[LimitId.PUBLIC_RESOURCES]: { value: 150, description: "Team limit" },
[LimitId.PRIVATE_RESOURCES]: { value: 150, description: "Team limit" },
[LimitId.MACHINE_CLIENTS]: { value: 25, description: "Team limit" }
};
export const tier3LimitSet: LimitSet = {
[FeatureId.USERS]: {
[LimitId.USERS]: {
value: 250,
description: "Business limit"
},
[FeatureId.SITES]: {
[LimitId.SITES]: {
value: 250,
description: "Business limit"
},
[FeatureId.DOMAINS]: {
[LimitId.DOMAINS]: {
value: 100,
description: "Business limit"
},
[FeatureId.REMOTE_EXIT_NODES]: {
[LimitId.REMOTE_EXIT_NODES]: {
value: 20,
description: "Business limit"
},
[FeatureId.ORGINIZATIONS]: {
[LimitId.ORGANIZATIONS]: {
value: 5,
description: "Business limit"
},
[LimitId.PUBLIC_RESOURCES]: { value: 750, description: "Business limit" },
[LimitId.PRIVATE_RESOURCES]: { value: 750, description: "Business limit" },
[LimitId.MACHINE_CLIENTS]: { value: 100, description: "Business limit" }
};
+2 -2
View File
@@ -1,7 +1,7 @@
import { db, limits } from "@server/db";
import { and, eq } from "drizzle-orm";
import { LimitSet } from "./limitSet";
import { FeatureId } from "./features";
import { LimitId } from "./features";
import logger from "@server/logger";
class LimitService {
@@ -38,7 +38,7 @@ class LimitService {
async getOrgLimit(
orgId: string,
featureId: FeatureId
featureId: LimitId
): Promise<number | null> {
const limitId = `${orgId}-${featureId}`;
const [limit] = await db
+11 -7
View File
@@ -16,20 +16,22 @@ export enum TierFeature {
SessionDurationPolicies = "sessionDurationPolicies", // handle downgrade by setting to default duration
PasswordExpirationPolicies = "passwordExpirationPolicies", // handle downgrade by setting to default duration
AutoProvisioning = "autoProvisioning", // handle downgrade by disabling auto provisioning
SshPam = "sshPam",
FullRbac = "fullRbac",
SiteProvisioningKeys = "siteProvisioningKeys", // handle downgrade by revoking keys if needed
SIEM = "siem", // handle downgrade by disabling SIEM integrations
HTTPPrivateResources = "httpPrivateResources", // handle downgrade by disabling HTTP private resources
DomainNamespaces = "domainNamespaces", // handle downgrade by removing custom domain namespaces
StandaloneHealthChecks = "standaloneHealthChecks",
AlertingRules = "alertingRules",
WildcardSubdomain = "wildcardSubdomain",
Labels = "labels"
Labels = "labels",
NewtAutoUpdate = "newtAutoUpdate",
ResourcePolicies = "resourcePolicies",
AdvancedPublicResources = "advancedPublicResources",
AdvancedPrivateResources = "advancedPrivateResources"
}
export const tierMatrix: Record<TierFeature, Tier[]> = {
[TierFeature.Labels]: ["tier2", "tier3", "enterprise"],
[TierFeature.Labels]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.OrgOidc]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.LoginPageDomain]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.DeviceApprovals]: ["tier1", "tier3", "enterprise"],
@@ -60,13 +62,15 @@ export const tierMatrix: Record<TierFeature, Tier[]> = {
"enterprise"
],
[TierFeature.AutoProvisioning]: ["tier1", "tier3", "enterprise"],
[TierFeature.SshPam]: ["tier1", "tier3", "enterprise"],
[TierFeature.FullRbac]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.SiteProvisioningKeys]: ["tier3", "enterprise"],
[TierFeature.SIEM]: ["enterprise"],
[TierFeature.HTTPPrivateResources]: ["tier3", "enterprise"],
[TierFeature.DomainNamespaces]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.StandaloneHealthChecks]: ["tier3", "enterprise"],
[TierFeature.AlertingRules]: ["tier3", "enterprise"],
[TierFeature.WildcardSubdomain]: ["tier1", "tier2", "tier3", "enterprise"]
[TierFeature.WildcardSubdomain]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.NewtAutoUpdate]: ["tier1", "tier2", "tier3", "enterprise"],
[TierFeature.ResourcePolicies]: ["tier3", "enterprise"],
[TierFeature.AdvancedPublicResources]: ["tier3", "enterprise"],
[TierFeature.AdvancedPrivateResources]: ["tier3", "enterprise"]
};
+26 -19
View File
@@ -9,10 +9,10 @@ import {
Transaction,
orgs
} from "@server/db";
import { FeatureId, getFeatureMeterId } from "./features";
import { LimitId, getFeatureMeterId } from "./features";
import logger from "@server/logger";
import { build } from "@server/build";
import cache from "#dynamic/lib/cache";
import { regionalCache as cache } from "#dynamic/lib/cache";
export function noop() {
if (build !== "saas") {
@@ -22,7 +22,6 @@ export function noop() {
}
export class UsageService {
constructor() {
if (noop()) {
return;
@@ -38,7 +37,7 @@ export class UsageService {
public async add(
orgId: string,
featureId: FeatureId,
featureId: LimitId,
value: number,
transaction: any = null
): Promise<Usage | null> {
@@ -57,7 +56,10 @@ export class UsageService {
try {
let usage;
if (transaction) {
const orgIdToUse = await this.getBillingOrg(orgId, transaction);
const orgIdToUse = await this.getBillingOrg(
orgId,
transaction
);
usage = await this.internalAddUsage(
orgIdToUse,
featureId,
@@ -112,7 +114,7 @@ export class UsageService {
private async internalAddUsage(
orgId: string, // here the orgId is the billing org already resolved by getBillingOrg in updateCount
featureId: FeatureId,
featureId: LimitId,
value: number,
trx: Transaction
): Promise<Usage> {
@@ -161,7 +163,7 @@ export class UsageService {
async updateCount(
orgId: string,
featureId: FeatureId,
featureId: LimitId,
value?: number,
customerId?: string
): Promise<void> {
@@ -225,7 +227,7 @@ export class UsageService {
private async getCustomerId(
orgId: string,
featureId: FeatureId
featureId: LimitId
): Promise<string | null> {
const orgIdToUse = await this.getBillingOrg(orgId);
@@ -267,18 +269,19 @@ export class UsageService {
public async getUsage(
orgId: string,
featureId: FeatureId,
featureId: LimitId,
trx: Transaction | typeof db = db
): Promise<Usage | null> {
if (noop()) {
return null;
}
const orgIdToUse = await this.getBillingOrg(orgId, trx);
const usageId = `${orgIdToUse}-${featureId}`;
let orgIdToUse = orgId;
try {
orgIdToUse = await this.getBillingOrg(orgId, trx);
const usageId = `${orgIdToUse}-${featureId}`;
const [result] = await trx
.select()
.from(usage)
@@ -338,8 +341,12 @@ export class UsageService {
`Failed to get usage for ${orgIdToUse}/${featureId}:`,
error
);
throw error;
if (process.env.NODE_ENV !== "development") {
throw error;
}
}
return null;
}
public async getBillingOrg(
@@ -374,7 +381,7 @@ export class UsageService {
public async checkLimitSet(
orgId: string,
featureId?: FeatureId,
featureId?: LimitId,
usage?: Usage,
trx: Transaction | typeof db = db
): Promise<boolean> {
@@ -382,13 +389,13 @@ export class UsageService {
return false;
}
const orgIdToUse = await this.getBillingOrg(orgId, trx);
// This method should check the current usage against the limits set for the organization
// and kick out all of the sites on the org
let hasExceededLimits = false;
let orgIdToUse = orgId;
try {
orgIdToUse = await this.getBillingOrg(orgId, trx);
let orgLimits: Limit[] = [];
if (featureId) {
// Get all limits set for this organization
@@ -422,7 +429,7 @@ export class UsageService {
} else {
currentUsage = await this.getUsage(
orgIdToUse,
limit.featureId as FeatureId,
limit.featureId as LimitId,
trx
);
}
+98 -200
View File
@@ -3,29 +3,43 @@ import {
newts,
blueprints,
Blueprint,
Site,
siteResources,
roleSiteResources,
userSiteResources,
clientSiteResources
} from "@server/db";
import { Config, ConfigSchema } from "./types";
import { ProxyResourcesResults, updateProxyResources } from "./proxyResources";
import { Config, ConfigSchema, isTargetsOnlyResource } from "./types";
import {
PublicResourcesResults,
updatePublicResources
} from "./publicResources";
import { fromError } from "zod-validation-error";
import logger from "@server/logger";
import { sites } from "@server/db";
import { eq, and, isNotNull } from "drizzle-orm";
import { addTargets as addProxyTargets } from "@server/routers/newt/targets";
import { addTargets as addClientTargets } from "@server/routers/client/targets";
import {
addTargets as addProxyTargets,
sendBrowserGatewayTargets
} from "@server/routers/newt/targets";
import {
ClientResourcesResults,
updateClientResources
} from "./clientResources";
updatePrivateResources
} from "./privateResources";
import { updateResourcePolicies } from "./resourcePolicies";
import { BlueprintSource } from "@server/routers/blueprints/types";
import { stringify as stringifyYaml } from "yaml";
import { faker } from "@faker-js/faker";
import { handleMessagingForUpdatedSiteResource } from "@server/routers/siteResource";
import { rebuildClientAssociationsFromSiteResource } from "../rebuildClientAssociations";
import { generateName } from "@server/db/names";
import {
handleMessagingForUpdatedSiteResource,
rebuildClientAssociationsFromSiteResource,
waitForSiteResourceRebuildIdle
} from "../rebuildClientAssociations";
import { build } from "@server/build";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
import next from "next";
import { LimitId } from "../billing";
import { usageService } from "../billing/usageService";
type ApplyBlueprintArgs = {
orgId: string;
@@ -42,40 +56,39 @@ export async function applyBlueprint({
name,
source = "API"
}: ApplyBlueprintArgs): Promise<Blueprint> {
// Validate the input data
const validationResult = ConfigSchema.safeParse(configData);
if (!validationResult.success) {
throw new Error(fromError(validationResult.error).toString());
}
const config: Config = validationResult.data;
let blueprintSucceeded: boolean = false;
let blueprintMessage: string;
let blueprintMessage = "";
let error: any | null = null;
try {
let proxyResourcesResults: ProxyResourcesResults = [];
let clientResourcesResults: ClientResourcesResults = [];
await db.transaction(async (trx) => {
proxyResourcesResults = await updateProxyResources(
orgId,
config,
trx,
siteId
);
clientResourcesResults = await updateClientResources(
orgId,
config,
trx,
siteId
);
const validationResult = ConfigSchema.safeParse(configData);
if (!validationResult.success) {
throw new Error(fromError(validationResult.error).toString());
}
logger.debug(
`Successfully updated proxy resources for org ${orgId}: ${JSON.stringify(proxyResourcesResults)}`
const config: Config = validationResult.data;
let publicResourcesResults: PublicResourcesResults = [];
let privateResourcesResults: ClientResourcesResults = [];
await db.transaction(async (trx) => {
await updateResourcePolicies(orgId, config, trx);
publicResourcesResults = await updatePublicResources(
orgId,
config,
trx,
siteId
);
privateResourcesResults = await updatePrivateResources(
orgId,
config,
trx,
siteId
);
// We need to update the targets on the newts from the successfully updated information
for (const result of proxyResourcesResults) {
for (const result of publicResourcesResults) {
for (const target of result.targetsToUpdate) {
const [site] = await trx
.select()
@@ -102,178 +115,63 @@ export async function applyBlueprint({
(hc) => hc.targetId === target.targetId
);
await addProxyTargets(
site.newt.newtId,
[target],
matchingHealthcheck ? [matchingHealthcheck] : [],
result.proxyResource.protocol,
site.newt.version
);
if (["http", "tcp", "udp"].includes(target.mode)) {
await addProxyTargets(
site.newt.newtId,
[target],
matchingHealthcheck
? [matchingHealthcheck]
: [],
result.proxyResource.mode === "udp"
? "udp"
: "tcp",
site.newt.version
);
} else if (
["ssh", "rdp", "vnc"].includes(target.mode)
) {
await sendBrowserGatewayTargets(
site.newt.newtId,
[target],
site.newt.version
);
}
}
}
}
logger.debug(
`Successfully updated client resources for org ${orgId}: ${JSON.stringify(clientResourcesResults)}`
`Successfully updated public resources for org ${orgId}: ${JSON.stringify(publicResourcesResults)}`
);
// We need to update the targets on the newts from the successfully updated information
for (const result of clientResourcesResults) {
if (
result.oldSiteResource &&
JSON.stringify(result.newSites?.sort()) !==
JSON.stringify(result.oldSites?.sort())
) {
// query existing associations
const existingRoleIds = await trx
.select()
.from(roleSiteResources)
.where(
eq(
roleSiteResources.siteResourceId,
result.oldSiteResource.siteResourceId
)
for (const result of privateResourcesResults) {
rebuildClientAssociationsFromSiteResource(
result.newSiteResource
)
.then(() =>
waitForSiteResourceRebuildIdle(
result.newSiteResource.siteResourceId
)
.then((rows) => rows.map((row) => row.roleId));
const existingUserIds = await trx
.select()
.from(userSiteResources)
.where(
eq(
userSiteResources.siteResourceId,
result.oldSiteResource.siteResourceId
)
)
.then(() =>
handleMessagingForUpdatedSiteResource(
result.oldSiteResource,
result.newSiteResource,
result.oldSites.map((s) => s.siteId),
result.newSites.map((s) => s.siteId)
)
.then((rows) => rows.map((row) => row.userId));
const existingClientIds = await trx
.select()
.from(clientSiteResources)
.where(
eq(
clientSiteResources.siteResourceId,
result.oldSiteResource.siteResourceId
)
)
.then((rows) => rows.map((row) => row.clientId));
// delete the existing site resource
await trx
.delete(siteResources)
.where(
and(
eq(
siteResources.siteResourceId,
result.oldSiteResource.siteResourceId
)
)
)
.catch((e) => {
logger.error(
`Failed to rebuild and handle messaging for site resource ${result.newSiteResource.siteResourceId}. Error: ${e}`
);
await rebuildClientAssociationsFromSiteResource(
result.oldSiteResource,
trx
);
const [insertedSiteResource] = await trx
.insert(siteResources)
.values({
...result.newSiteResource
})
.returning();
// wait some time to allow for messages to be handled
await new Promise((resolve) => setTimeout(resolve, 750));
//////////////////// update the associations ////////////////////
if (existingRoleIds.length > 0) {
await trx.insert(roleSiteResources).values(
existingRoleIds.map((roleId) => ({
roleId,
siteResourceId:
insertedSiteResource!.siteResourceId
}))
);
}
if (existingUserIds.length > 0) {
await trx.insert(userSiteResources).values(
existingUserIds.map((userId) => ({
userId,
siteResourceId:
insertedSiteResource!.siteResourceId
}))
);
}
if (existingClientIds.length > 0) {
await trx.insert(clientSiteResources).values(
existingClientIds.map((clientId) => ({
clientId,
siteResourceId:
insertedSiteResource!.siteResourceId
}))
);
}
await rebuildClientAssociationsFromSiteResource(
insertedSiteResource,
trx
);
} else {
let good = true;
for (const newSite of result.newSites) {
const [site] = await trx
.select()
.from(sites)
.innerJoin(newts, eq(sites.siteId, newts.siteId))
.where(
and(
eq(sites.siteId, newSite.siteId),
eq(sites.orgId, orgId),
eq(sites.type, "newt"),
isNotNull(sites.pubKey)
)
)
.limit(1);
if (!site) {
logger.debug(
`No newt sites found for client resource ${result.newSiteResource.siteResourceId}, skipping target update`
);
good = false;
break;
}
logger.debug(
`Updating client resource ${result.newSiteResource.siteResourceId} on site ${newSite.siteId}`
);
}
if (!good) {
continue;
}
await handleMessagingForUpdatedSiteResource(
result.oldSiteResource,
result.newSiteResource,
result.newSites.map((site) => ({
siteId: site.siteId,
orgId: result.newSiteResource.orgId
})),
trx
);
}
// await addClientTargets(
// site.newt.newtId,
// result.resource.destination,
// result.resource.destinationPort,
// result.resource.protocol,
// result.resource.proxyPort
// );
});
}
logger.debug(
`Successfully updated private resources for org ${orgId}: ${JSON.stringify(privateResourcesResults)}`
);
});
blueprintSucceeded = true;
@@ -281,7 +179,9 @@ export async function applyBlueprint({
} catch (err) {
blueprintSucceeded = false;
blueprintMessage = `Blueprint applied with errors: ${err}`;
logger.error(blueprintMessage);
logger.debug(
`Org ${orgId} blueprint apply issues: ${blueprintMessage}`
);
error = err;
}
@@ -291,9 +191,7 @@ export async function applyBlueprint({
.insert(blueprints)
.values({
orgId,
name:
name ??
`${faker.word.adjective()}-${faker.word.adjective()}-${faker.word.noun()}`,
name: name ?? generateName(),
contents: stringifyYaml(configData),
createdAt: Math.floor(Date.now() / 1000),
succeeded: blueprintSucceeded,
@@ -1,10 +1,56 @@
import { sendToClient } from "#dynamic/routers/ws";
import { processContainerLabels } from "./parseDockerContainers";
import { applyBlueprint } from "./applyBlueprint";
import { PrivateResourceSchema, PublicResourceSchema } from "./types";
import { db, sites } from "@server/db";
import { eq } from "drizzle-orm";
import logger from "@server/logger";
type BlueprintResult = ReturnType<typeof processContainerLabels>;
function filterInvalidResources(blueprint: BlueprintResult): {
skippedCount: number;
skippedKeys: string[];
} {
const skippedKeys: string[] = [];
for (const section of ["proxy-resources", "public-resources"] as const) {
const resources = blueprint[section];
for (const [key, value] of Object.entries(resources)) {
const result = PublicResourceSchema.safeParse(value);
if (!result.success) {
const errors = result.error.issues
.map((i) => `${i.path.join(".")}: ${i.message}`)
.join("; ");
logger.warn(
`Skipping invalid Docker ${section} "${key}": ${errors}`
);
delete resources[key];
skippedKeys.push(`${section}.${key}`);
}
}
}
for (const section of ["client-resources", "private-resources"] as const) {
const resources = blueprint[section];
for (const [key, value] of Object.entries(resources)) {
const result = PrivateResourceSchema.safeParse(value);
if (!result.success) {
const errors = result.error.issues
.map((i) => `${i.path.join(".")}: ${i.message}`)
.join("; ");
logger.warn(
`Skipping invalid Docker ${section} "${key}": ${errors}`
);
delete resources[key];
skippedKeys.push(`${section}.${key}`);
}
}
}
return { skippedCount: skippedKeys.length, skippedKeys };
}
export async function applyNewtDockerBlueprint(
siteId: number,
newtId: string,
@@ -21,17 +67,27 @@ export async function applyNewtDockerBlueprint(
return;
}
// logger.debug(`Applying Docker blueprint to site: ${siteId}`);
// logger.debug(`Containers: ${JSON.stringify(containers, null, 2)}`);
let skippedCount = 0;
let skippedKeys: string[] = [];
try {
const blueprint = processContainerLabels(containers);
// Some Newt clients can report null/undefined containers when Docker
// labels are unavailable. Treat that as an empty blueprint payload.
const safeContainers = Array.isArray(containers) ? containers : [];
const blueprint = processContainerLabels(safeContainers);
logger.debug(`Received Docker blueprint: ${JSON.stringify(blueprint)}`);
logger.debug(
`Received Docker blueprint with ${Object.keys(blueprint["proxy-resources"]).length} proxy, ${Object.keys(blueprint["client-resources"]).length} client resource(s)`
);
// make sure this is not an empty object
if (isEmptyObject(blueprint)) {
return;
const filterResult = filterInvalidResources(blueprint);
skippedCount = filterResult.skippedCount;
skippedKeys = filterResult.skippedKeys;
if (skippedCount > 0) {
logger.warn(
`Filtered ${skippedCount} invalid resource(s) from Docker blueprint: ${skippedKeys.join(", ")}`
);
}
if (
@@ -40,6 +96,15 @@ export async function applyNewtDockerBlueprint(
isEmptyObject(blueprint["public-resources"]) &&
isEmptyObject(blueprint["private-resources"])
) {
if (skippedCount > 0) {
await sendToClient(newtId, {
type: "newt/blueprint/results",
data: {
success: false,
message: `All resources were invalid and skipped: ${skippedKeys.join(", ")}`
}
});
}
return;
}
@@ -51,7 +116,7 @@ export async function applyNewtDockerBlueprint(
source: "NEWT"
});
} catch (error) {
logger.error(`Failed to update database from config: ${error}`);
logger.debug(`Failed to update database from config: ${error}`);
await sendToClient(newtId, {
type: "newt/blueprint/results",
data: {
@@ -66,7 +131,10 @@ export async function applyNewtDockerBlueprint(
type: "newt/blueprint/results",
data: {
success: true,
message: "Config updated successfully"
message:
skippedCount > 0
? `Config updated successfully. Skipped ${skippedCount} invalid resource(s): ${skippedKeys.join(", ")}`
: "Config updated successfully"
}
});
}
@@ -3,6 +3,7 @@ import {
clientSiteResources,
domains,
orgDomains,
roleActions,
roles,
roleSiteResources,
Site,
@@ -19,8 +20,17 @@ import { sites } from "@server/db";
import { eq, and, ne, inArray, or, isNotNull } from "drizzle-orm";
import { Config } from "./types";
import logger from "@server/logger";
import { defaultRoleAllowedActions } from "@server/routers/role/createRole";
import { getNextAvailableAliasAddress } from "../ip";
import { createCertificate } from "#dynamic/routers/certificates/createCertificate";
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
import { tierMatrix } from "../billing/tierMatrix";
import { build } from "@server/build";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
import next from "next";
import { LimitId } from "../billing";
import { usageService } from "../billing/usageService";
async function getDomainForSiteResource(
siteResourceId: number | undefined,
@@ -101,7 +111,7 @@ export type ClientResourcesResults = {
oldSites: { siteId: number }[];
}[];
export async function updateClientResources(
export async function updatePrivateResources(
orgId: string,
config: Config,
trx: Transaction,
@@ -112,6 +122,30 @@ export async function updateClientResources(
for (const [resourceNiceId, resourceData] of Object.entries(
config["client-resources"]
)) {
if (resourceData.mode === "http") {
const hasHttpFeature = await isLicensedOrSubscribed(
orgId,
tierMatrix.advancedPrivateResources
);
if (!hasHttpFeature) {
throw new Error(
"HTTP private resources are not included in your current plan. Please upgrade."
);
}
}
if (resourceData.mode === "ssh") {
const hasSshFeature = await isLicensedOrSubscribed(
orgId,
tierMatrix.advancedPrivateResources
);
if (!hasSshFeature) {
throw new Error(
"SSH private resources are not included in your current plan. Please upgrade."
);
}
}
const [existingResource] = await trx
.select()
.from(siteResources)
@@ -225,7 +259,11 @@ export async function updateClientResources(
: resourceData["udp-ports"],
fullDomain: resourceData["full-domain"] || null,
subdomain: domainInfo ? domainInfo.subdomain : null,
domainId: domainInfo ? domainInfo.domainId : null
domainId: domainInfo ? domainInfo.domainId : null,
pamMode: resourceData["auth-daemon"]?.pam || "passthrough",
authDaemonMode:
resourceData["auth-daemon"]?.mode || "native",
authDaemonPort: resourceData["auth-daemon"]?.port || 22123
})
.where(
eq(
@@ -332,8 +370,7 @@ export async function updateClientResources(
}
if (resourceData.roles.length > 0) {
// Re-add specified roles but we need to get the roleIds from the role name in the array
const rolesToUpdate = await trx
const existingRoles = await trx
.select()
.from(roles)
.where(
@@ -343,7 +380,30 @@ export async function updateClientResources(
)
);
const roleIds = rolesToUpdate.map((role) => role.roleId);
const foundNames = new Set(existingRoles.map((r) => r.name));
const missingNames = resourceData.roles.filter(
(n) => !foundNames.has(n)
);
for (const name of missingNames) {
const [created] = await trx
.insert(roles)
.values({ name, orgId })
.returning();
await trx.insert(roleActions).values(
defaultRoleAllowedActions.map((action) => ({
roleId: created.roleId,
actionId: action,
orgId
}))
);
existingRoles.push(created);
logger.info(
`Auto-created role "${name}" in org ${orgId} from blueprint`
);
}
const roleIds = existingRoles.map((role) => role.roleId);
await trx
.insert(roleSiteResources)
@@ -359,9 +419,47 @@ export async function updateClientResources(
oldSites: existingSiteIds
});
} else {
// create a brand new resource
if (build == "saas") {
const usage = await usageService.getUsage(
orgId,
LimitId.PRIVATE_RESOURCES
);
if (!usage) {
throw new Error(
`Usage data not found for org ${orgId} and limit ${LimitId.PRIVATE_RESOURCES}`
);
}
const rejectResource = await usageService.checkLimitSet(
orgId,
LimitId.PRIVATE_RESOURCES,
{
...usage,
instantaneousValue: (usage.instantaneousValue || 0) + 1
} // We need to add one to know if we are violating the limit
);
if (rejectResource) {
throw new Error(
"Private resource limit exceeded. Please upgrade your plan."
);
}
}
let aliasAddress: string | null = null;
if (resourceData.mode === "host" || resourceData.mode === "http") {
aliasAddress = await getNextAvailableAliasAddress(orgId, trx);
let releaseAliasLock: (() => Promise<void>) | null = null;
if (
resourceData.mode === "host" ||
resourceData.mode === "http" ||
resourceData.mode === "ssh"
) {
const { value, release } = await getNextAvailableAliasAddress(
orgId,
trx
);
aliasAddress = value;
releaseAliasLock = release;
}
let domainInfo:
@@ -415,10 +513,16 @@ export async function updateClientResources(
: resourceData["udp-ports"],
fullDomain: resourceData["full-domain"] || null,
subdomain: domainInfo ? domainInfo.subdomain : null,
domainId: domainInfo ? domainInfo.domainId : null
domainId: domainInfo ? domainInfo.domainId : null,
pamMode: resourceData["auth-daemon"]?.pam || "passthrough",
authDaemonMode:
resourceData["auth-daemon"]?.mode || "native",
authDaemonPort: resourceData["auth-daemon"]?.port || 22123
})
.returning();
await releaseAliasLock?.();
const siteResourceId = newResource.siteResourceId;
for (const site of allSites) {
@@ -444,8 +548,7 @@ export async function updateClientResources(
});
if (resourceData.roles.length > 0) {
// get roleIds from role names
const rolesToUpdate = await trx
const existingRoles = await trx
.select()
.from(roles)
.where(
@@ -455,7 +558,30 @@ export async function updateClientResources(
)
);
const roleIds = rolesToUpdate.map((role) => role.roleId);
const foundNames = new Set(existingRoles.map((r) => r.name));
const missingNames = resourceData.roles.filter(
(n) => !foundNames.has(n)
);
for (const name of missingNames) {
const [created] = await trx
.insert(roles)
.values({ name, orgId })
.returning();
await trx.insert(roleActions).values(
defaultRoleAllowedActions.map((action) => ({
roleId: created.roleId,
actionId: action,
orgId
}))
);
existingRoles.push(created);
logger.info(
`Auto-created role "${name}" in org ${orgId} from blueprint`
);
}
const roleIds = existingRoles.map((role) => role.roleId);
await trx
.insert(roleSiteResources)
@@ -517,6 +643,8 @@ export async function updateClientResources(
`Created new client resource ${newResource.name} (${newResource.siteResourceId}) for org ${orgId}`
);
await usageService.add(orgId, LimitId.PRIVATE_RESOURCES, 1, trx);
results.push({
newSiteResource: newResource,
newSites: allSites,
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+653
View File
@@ -0,0 +1,653 @@
import {
db,
idp,
idpOrg,
resourcePolicies,
resourcePolicyHeaderAuth,
resourcePolicyPassword,
resourcePolicyPincode,
resourcePolicyRules,
resourcePolicyWhiteList,
rolePolicies,
roles,
Transaction,
userOrgs,
userPolicies,
users
} from "@server/db";
import { eq, and, or } from "drizzle-orm";
import { Config, ResourcePolicyData } from "./types";
import logger from "@server/logger";
import { getUniqueResourcePolicyName } from "@server/db/names";
import { hashPassword } from "@server/auth/password";
import { isValidCIDR, isValidIP, isValidUrlGlobPattern } from "../validators";
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
import { tierMatrix } from "../billing/tierMatrix";
export type ResourcePoliciesResults = {
resourcePolicyId: number;
niceId: string;
}[];
export async function updateResourcePolicies(
orgId: string,
config: Config,
trx: Transaction
): Promise<ResourcePoliciesResults> {
const results: ResourcePoliciesResults = [];
for (const [policyNiceId, policyData] of Object.entries(
config["public-policies"]
)) {
const isLicensed = await isLicensedOrSubscribed(
orgId,
tierMatrix.resourcePolicies
);
if (!isLicensed) {
throw new Error(
"Your current subscription does not support shared resource policies. Please upgrade to access this feature."
);
}
// Validate rules
for (const rule of policyData.rules) {
if (rule.match === "cidr" && !isValidCIDR(rule.value)) {
throw new Error(
`Invalid CIDR provided in resource policy '${policyNiceId}': ${rule.value}`
);
} else if (rule.match === "ip" && !isValidIP(rule.value)) {
throw new Error(
`Invalid IP provided in resource policy '${policyNiceId}': ${rule.value}`
);
} else if (
rule.match === "path" &&
!isValidUrlGlobPattern(rule.value)
) {
throw new Error(
`Invalid URL glob pattern provided in resource policy '${policyNiceId}': ${rule.value}`
);
}
}
// Validate auto-login-idp if provided
if (policyData["auto-login-idp"]) {
const [provider] = await trx
.select()
.from(idp)
.innerJoin(idpOrg, eq(idpOrg.idpId, idp.idpId))
.where(
and(
eq(idp.idpId, policyData["auto-login-idp"]),
eq(idpOrg.orgId, orgId)
)
)
.limit(1);
if (!provider) {
throw new Error(
`Identity provider not found for policy '${policyNiceId}' in this organization`
);
}
}
// Look up the admin role
const [adminRole] = await trx
.select()
.from(roles)
.where(and(eq(roles.isAdmin, true), eq(roles.orgId, orgId)))
.limit(1);
if (!adminRole) {
throw new Error("Admin role not found");
}
// Find existing policy by niceId and orgId
const [existingPolicy] = await trx
.select()
.from(resourcePolicies)
.where(
and(
eq(resourcePolicies.niceId, policyNiceId),
eq(resourcePolicies.orgId, orgId)
)
)
.limit(1);
let resourcePolicyId: number;
if (existingPolicy) {
// Update the existing policy
await trx
.update(resourcePolicies)
.set({
name: policyData.name,
sso: policyData.sso ?? true,
idpId: policyData["auto-login-idp"] ?? null,
emailWhitelistEnabled:
policyData["email-whitelist-enabled"] ??
policyData["whitelist-users"].length > 0,
applyRules:
policyData["apply-rules"] || policyData.rules.length > 0
})
.where(
eq(
resourcePolicies.resourcePolicyId,
existingPolicy.resourcePolicyId
)
);
resourcePolicyId = existingPolicy.resourcePolicyId;
// Sync password
await trx
.delete(resourcePolicyPassword)
.where(
eq(
resourcePolicyPassword.resourcePolicyId,
resourcePolicyId
)
);
if (policyData.password) {
const passwordHash = await hashPassword(policyData.password);
await trx.insert(resourcePolicyPassword).values({
resourcePolicyId,
passwordHash
});
}
// Sync pincode
await trx
.delete(resourcePolicyPincode)
.where(
eq(resourcePolicyPincode.resourcePolicyId, resourcePolicyId)
);
if (policyData.pincode) {
const pincodeHash = await hashPassword(policyData.pincode);
await trx.insert(resourcePolicyPincode).values({
resourcePolicyId,
pincodeHash,
digitLength: 6
});
}
// Sync header auth
await trx
.delete(resourcePolicyHeaderAuth)
.where(
eq(
resourcePolicyHeaderAuth.resourcePolicyId,
resourcePolicyId
)
);
if (policyData["basic-auth"]) {
const basicAuth = policyData["basic-auth"];
const headerAuthHash = await hashPassword(
Buffer.from(
`${basicAuth.user}:${basicAuth.password}`
).toString("base64")
);
await trx.insert(resourcePolicyHeaderAuth).values({
resourcePolicyId,
headerAuthHash,
extendedCompatibility:
basicAuth["extended-compatibility"] ?? true
});
}
// Sync SSO roles
await syncRolePolicies(
resourcePolicyId,
policyData["sso-roles"],
orgId,
adminRole.roleId,
trx
);
// Sync SSO users
await syncUserPolicies(
resourcePolicyId,
policyData["sso-users"],
orgId,
trx
);
// Sync whitelist users
await syncWhitelistPolicyUsers(
resourcePolicyId,
policyData["whitelist-users"],
trx
);
// Sync rules
await syncPolicyRules(resourcePolicyId, policyData.rules, trx);
logger.debug(
`Updated resource policy ${resourcePolicyId} (${policyNiceId})`
);
} else {
// Create a new policy
const [newPolicy] = await trx
.insert(resourcePolicies)
.values({
niceId: policyNiceId,
orgId,
name: policyData.name,
sso: policyData.sso ?? true,
idpId: policyData["auto-login-idp"] ?? null,
emailWhitelistEnabled:
policyData["email-whitelist-enabled"] ??
policyData["whitelist-users"].length > 0,
applyRules:
policyData["apply-rules"] ||
policyData.rules.length > 0,
scope: "global"
})
.returning();
resourcePolicyId = newPolicy.resourcePolicyId;
// Always add admin role
await trx.insert(rolePolicies).values({
roleId: adminRole.roleId,
resourcePolicyId
});
// Add SSO roles
await addRolePolicies(
resourcePolicyId,
policyData["sso-roles"],
orgId,
adminRole.roleId,
trx
);
// Add SSO users
await addUserPolicies(
resourcePolicyId,
policyData["sso-users"],
orgId,
trx
);
// Add password
if (policyData.password) {
const passwordHash = await hashPassword(policyData.password);
await trx.insert(resourcePolicyPassword).values({
resourcePolicyId,
passwordHash
});
}
// Add pincode
if (policyData.pincode) {
const pincodeHash = await hashPassword(policyData.pincode);
await trx.insert(resourcePolicyPincode).values({
resourcePolicyId,
pincodeHash,
digitLength: 6
});
}
// Add header auth
if (policyData["basic-auth"]) {
const basicAuth = policyData["basic-auth"];
const headerAuthHash = await hashPassword(
Buffer.from(
`${basicAuth.user}:${basicAuth.password}`
).toString("base64")
);
await trx.insert(resourcePolicyHeaderAuth).values({
resourcePolicyId,
headerAuthHash,
extendedCompatibility:
basicAuth["extended-compatibility"] ?? true
});
}
// Add whitelist users
if (policyData["whitelist-users"].length > 0) {
await trx.insert(resourcePolicyWhiteList).values(
policyData["whitelist-users"].map((email) => ({
email,
resourcePolicyId
}))
);
}
// Add rules
if (policyData.rules.length > 0) {
await trx.insert(resourcePolicyRules).values(
policyData.rules.map((rule, index) => ({
resourcePolicyId,
action: getRuleAction(rule.action),
match: getRuleMatch(rule.match),
value: rule.value,
priority: rule.priority ?? index + 1,
enabled: rule.enabled ?? true
}))
);
}
logger.debug(
`Created resource policy ${resourcePolicyId} (${policyNiceId})`
);
}
results.push({ resourcePolicyId, niceId: policyNiceId });
}
return results;
}
function getRuleAction(input: string): "ACCEPT" | "DROP" | "PASS" {
if (input === "allow") return "ACCEPT";
if (input === "deny") return "DROP";
return "PASS";
}
function getRuleMatch(
input: string
): "CIDR" | "IP" | "PATH" | "COUNTRY" | "ASN" | "REGION" {
return input.toUpperCase() as
| "CIDR"
| "IP"
| "PATH"
| "COUNTRY"
| "ASN"
| "REGION";
}
async function syncRolePolicies(
policyId: number,
ssoRoles: string[],
orgId: string,
adminRoleId: number,
trx: Transaction
) {
const existingRolePolicies = await trx
.select()
.from(rolePolicies)
.where(eq(rolePolicies.resourcePolicyId, policyId));
for (const roleName of ssoRoles) {
const [role] = await trx
.select()
.from(roles)
.where(and(eq(roles.name, roleName), eq(roles.orgId, orgId)))
.limit(1);
if (!role) {
logger.warn(
`Role '${roleName}' not found in org '${orgId}', skipping`
);
continue;
}
if (role.isAdmin) {
continue; // admin role is always included, skip
}
const alreadyExists = existingRolePolicies.some(
(rp) => rp.roleId === role.roleId
);
if (!alreadyExists) {
await trx.insert(rolePolicies).values({
roleId: role.roleId,
resourcePolicyId: policyId
});
}
}
// Remove roles no longer in the list (except admin)
for (const existingRolePolicy of existingRolePolicies) {
if (existingRolePolicy.roleId === adminRoleId) {
continue;
}
const [role] = await trx
.select()
.from(roles)
.where(eq(roles.roleId, existingRolePolicy.roleId))
.limit(1);
if (role?.isAdmin) {
continue;
}
if (role && !ssoRoles.includes(role.name)) {
await trx
.delete(rolePolicies)
.where(
and(
eq(rolePolicies.resourcePolicyId, policyId),
eq(rolePolicies.roleId, existingRolePolicy.roleId)
)
);
}
}
}
async function addRolePolicies(
policyId: number,
ssoRoles: string[],
orgId: string,
adminRoleId: number,
trx: Transaction
) {
for (const roleName of ssoRoles) {
const [role] = await trx
.select()
.from(roles)
.where(and(eq(roles.name, roleName), eq(roles.orgId, orgId)))
.limit(1);
if (!role) {
logger.warn(
`Role '${roleName}' not found in org '${orgId}', skipping`
);
continue;
}
if (role.isAdmin) {
continue; // admin already added
}
await trx.insert(rolePolicies).values({
roleId: role.roleId,
resourcePolicyId: policyId
});
}
}
async function syncUserPolicies(
policyId: number,
ssoUsers: string[],
orgId: string,
trx: Transaction
) {
const existingUserPolicies = await trx
.select()
.from(userPolicies)
.where(eq(userPolicies.resourcePolicyId, policyId));
for (const username of ssoUsers) {
const [user] = await trx
.select()
.from(users)
.innerJoin(userOrgs, eq(users.userId, userOrgs.userId))
.where(
and(
or(eq(users.username, username), eq(users.email, username)),
eq(userOrgs.orgId, orgId)
)
)
.limit(1);
if (!user) {
logger.warn(
`User '${username}' not found in org '${orgId}', skipping`
);
continue;
}
const alreadyExists = existingUserPolicies.some(
(up) => up.userId === user.user.userId
);
if (!alreadyExists) {
await trx.insert(userPolicies).values({
userId: user.user.userId,
resourcePolicyId: policyId
});
}
}
// Remove users no longer in the list
for (const existingUserPolicy of existingUserPolicies) {
const [user] = await trx
.select()
.from(users)
.innerJoin(userOrgs, eq(users.userId, userOrgs.userId))
.where(
and(
eq(users.userId, existingUserPolicy.userId),
eq(userOrgs.orgId, orgId)
)
)
.limit(1);
if (
user &&
user.user.username &&
!ssoUsers.includes(user.user.username) &&
!ssoUsers.includes(user.user.email ?? "")
) {
await trx
.delete(userPolicies)
.where(
and(
eq(userPolicies.resourcePolicyId, policyId),
eq(userPolicies.userId, existingUserPolicy.userId)
)
);
}
}
}
async function addUserPolicies(
policyId: number,
ssoUsers: string[],
orgId: string,
trx: Transaction
) {
for (const username of ssoUsers) {
const [user] = await trx
.select()
.from(users)
.innerJoin(userOrgs, eq(users.userId, userOrgs.userId))
.where(
and(
or(eq(users.username, username), eq(users.email, username)),
eq(userOrgs.orgId, orgId)
)
)
.limit(1);
if (!user) {
logger.warn(
`User '${username}' not found in org '${orgId}', skipping`
);
continue;
}
await trx.insert(userPolicies).values({
userId: user.user.userId,
resourcePolicyId: policyId
});
}
}
async function syncWhitelistPolicyUsers(
policyId: number,
whitelistUsers: string[],
trx: Transaction
) {
const existingWhitelist = await trx
.select()
.from(resourcePolicyWhiteList)
.where(eq(resourcePolicyWhiteList.resourcePolicyId, policyId));
for (const email of whitelistUsers) {
const alreadyExists = existingWhitelist.some((w) => w.email === email);
if (!alreadyExists) {
await trx.insert(resourcePolicyWhiteList).values({
email,
resourcePolicyId: policyId
});
}
}
for (const existingEntry of existingWhitelist) {
if (!whitelistUsers.includes(existingEntry.email)) {
await trx
.delete(resourcePolicyWhiteList)
.where(
and(
eq(resourcePolicyWhiteList.resourcePolicyId, policyId),
eq(resourcePolicyWhiteList.email, existingEntry.email)
)
);
}
}
}
async function syncPolicyRules(
policyId: number,
rules: ResourcePolicyData["rules"],
trx: Transaction
) {
const existingRules = await trx
.select()
.from(resourcePolicyRules)
.where(eq(resourcePolicyRules.resourcePolicyId, policyId))
.orderBy(resourcePolicyRules.priority);
for (const [index, rule] of rules.entries()) {
const intendedPriority = rule.priority ?? index + 1;
const existingRule = existingRules[index];
if (existingRule) {
await trx
.update(resourcePolicyRules)
.set({
action: getRuleAction(rule.action),
match: getRuleMatch(rule.match),
value: rule.value,
priority: intendedPriority,
enabled: rule.enabled ?? true
})
.where(eq(resourcePolicyRules.ruleId, existingRule.ruleId));
} else {
await trx.insert(resourcePolicyRules).values({
resourcePolicyId: policyId,
action: getRuleAction(rule.action),
match: getRuleMatch(rule.match),
value: rule.value,
priority: intendedPriority,
enabled: rule.enabled ?? true
});
}
}
// Remove extra rules
if (existingRules.length > rules.length) {
const rulesToDelete = existingRules.slice(rules.length);
for (const rule of rulesToDelete) {
await trx
.delete(resourcePolicyRules)
.where(eq(resourcePolicyRules.ruleId, rule.ruleId));
}
}
}
+256 -34
View File
@@ -1,8 +1,23 @@
import { z } from "zod";
import { existsSync } from "node:fs";
import { portRangeStringSchema } from "@server/lib/ip";
import { MaintenanceSchema } from "#dynamic/lib/blueprints/MaintenanceSchema";
import { isValidRegionId } from "@server/db/regions";
import { wildcardSubdomainSchema } from "@server/lib/schemas";
import config from "@server/lib/config";
const maxmindDbPath = config.getRawConfig().server.maxmind_db_path;
const maxmindAsnPath = config.getRawConfig().server.maxmind_asn_path;
const hasMaxmindCountryDb =
typeof maxmindDbPath === "string" &&
maxmindDbPath.length > 0 &&
existsSync(maxmindDbPath);
const hasMaxmindAsnDb =
typeof maxmindAsnPath === "string" &&
maxmindAsnPath.length > 0 &&
existsSync(maxmindAsnPath);
export const SiteSchema = z.object({
name: z.string().min(1).max(100),
@@ -82,8 +97,9 @@ export const RuleSchema = z
.object({
action: z.enum(["allow", "deny", "pass"]),
match: z.enum(["cidr", "path", "ip", "country", "asn", "region"]),
value: z.string(),
priority: z.int().optional()
value: z.coerce.string(),
priority: z.int().optional(),
enabled: z.boolean().optional().default(true)
})
.refine(
(rule) => {
@@ -116,6 +132,9 @@ export const RuleSchema = z
.refine(
(rule) => {
if (rule.match === "country") {
if (!hasMaxmindCountryDb) {
return false;
}
// Check if it's a valid 2-letter country code or "ALL"
return /^[A-Z]{2}$/.test(rule.value) || rule.value === "ALL";
}
@@ -124,12 +143,15 @@ export const RuleSchema = z
{
path: ["value"],
message:
"Value must be a 2-letter country code or 'ALL' when match is 'country'"
"Country rules require a valid existing server.maxmind_db_path and value must be a 2-letter country code or 'ALL'"
}
)
.refine(
(rule) => {
if (rule.match === "asn") {
if (!hasMaxmindCountryDb || !hasMaxmindAsnDb) {
return false;
}
// Check if it's either AS<number> format or "ALL"
const asNumberPattern = /^AS\d+$/i;
return asNumberPattern.test(rule.value) || rule.value === "ALL";
@@ -139,7 +161,7 @@ export const RuleSchema = z
{
path: ["value"],
message:
"Value must be 'AS<number>' format or 'ALL' when match is 'asn'"
"ASN rules require valid existing server.maxmind_db_path and server.maxmind_asn_path, and value must be 'AS<number>' format or 'ALL'"
}
)
.refine(
@@ -161,11 +183,34 @@ export const HeaderSchema = z.object({
value: z.string().min(1)
});
export const AuthDaemonSchema = z
.object({
pam: z.enum(["passthrough", "push"]).optional().default("passthrough"),
mode: z.enum(["site", "remote", "native"]).optional().default("site"),
port: z.int().min(1).max(65535).optional()
})
.refine(
(data) => {
if (data.mode === "remote") {
return data.port !== undefined;
}
return true;
},
{
path: ["port"],
message: "port is required when auth-daemon mode is 'remote'"
}
);
// Schema for individual resource
export const ResourceSchema = z
export const PublicResourceSchema = z
.object({
name: z.string().optional(),
protocol: z.enum(["http", "tcp", "udp"]).optional(),
protocol: z
.enum(["http", "tcp", "udp", "ssh", "rdp", "vnc"])
.optional(), // this was the old one and is now DEPRECATED in favor of the mode
mode: z.enum(["http", "tcp", "udp", "ssh", "rdp", "vnc"]).optional(),
policy: z.string().optional(),
ssl: z.boolean().optional(),
scheme: z.enum(["http", "https"]).optional(),
"full-domain": z.string().optional(),
@@ -179,7 +224,10 @@ export const ResourceSchema = z
requestHeaders: z.array(HeaderSchema).optional(),
responseHeaders: z.array(HeaderSchema).optional(),
rules: z.array(RuleSchema).optional(),
maintenance: MaintenanceSchema.optional()
maintenance: MaintenanceSchema.optional(),
"auth-daemon": AuthDaemonSchema.optional(),
"proxy-protocol": z.boolean().optional(),
"proxy-protocol-version": z.int().min(1).optional()
})
.refine(
(resource) => {
@@ -187,9 +235,10 @@ export const ResourceSchema = z
return true;
}
// Otherwise, require name and protocol for full resource definition
// Otherwise, require name and protocol/mode for full resource definition
return (
resource.name !== undefined && resource.protocol !== undefined
resource.name !== undefined &&
(resource.mode !== undefined || resource.protocol !== undefined)
);
},
{
@@ -203,8 +252,8 @@ export const ResourceSchema = z
return true;
}
// If protocol is http, all targets must have method field
if (resource.protocol === "http") {
// If protocol/mode is http, all targets must have method field
if ((resource.mode ?? resource.protocol) === "http") {
return resource.targets.every(
(target) => target == null || target.method !== undefined
);
@@ -222,8 +271,9 @@ export const ResourceSchema = z
return true;
}
// If protocol is tcp or udp, no target should have method field
if (resource.protocol === "tcp" || resource.protocol === "udp") {
// If protocol/mode is tcp or udp, no target should have method field
const effectiveProtocol1 = resource.mode ?? resource.protocol;
if (effectiveProtocol1 === "tcp" || effectiveProtocol1 === "udp") {
return resource.targets.every(
(target) => target == null || target.method === undefined
);
@@ -241,8 +291,37 @@ export const ResourceSchema = z
return true;
}
// If protocol is http, it must have a full-domain
if (resource.protocol === "http") {
const effectiveProtocol = resource.mode ?? resource.protocol;
if (effectiveProtocol !== "ssh") {
return true;
}
const authDaemonMode = resource["auth-daemon"]?.mode;
if (authDaemonMode !== "native" && authDaemonMode !== "site") {
return true;
}
return (
resource.targets.filter((target) => target != null).length <= 1
);
},
{
path: ["targets"],
error: "When protocol is 'ssh' and auth-daemon mode is 'native' or 'site', only one target/site is allowed"
}
)
.refine(
(resource) => {
if (isTargetsOnlyResource(resource)) {
return true;
}
// If protocol/mode is http, ssh, rdp, or vnc, it must have a full-domain
const effectiveProtocol = resource.mode ?? resource.protocol;
if (
effectiveProtocol !== undefined &&
["http", "ssh", "rdp", "vnc"].includes(effectiveProtocol)
) {
return (
resource["full-domain"] !== undefined &&
resource["full-domain"].length > 0
@@ -252,7 +331,7 @@ export const ResourceSchema = z
},
{
path: ["full-domain"],
error: "When protocol is 'http', a 'full-domain' must be provided"
error: "When protocol is 'http', 'ssh', 'rdp', or 'vnc', a 'full-domain' must be provided"
}
)
.refine(
@@ -261,8 +340,9 @@ export const ResourceSchema = z
return true;
}
// If protocol is tcp or udp, it must have both proxy-port
if (resource.protocol === "tcp" || resource.protocol === "udp") {
// If protocol/mode is tcp or udp, it must have both proxy-port
const effectiveProtocol2 = resource.mode ?? resource.protocol;
if (effectiveProtocol2 === "tcp" || effectiveProtocol2 === "udp") {
return resource["proxy-port"] !== undefined;
}
return true;
@@ -279,8 +359,9 @@ export const ResourceSchema = z
return true;
}
// If protocol is tcp or udp, it must not have auth
if (resource.protocol === "tcp" || resource.protocol === "udp") {
// If protocol/mode is tcp or udp, it must not have auth
const effectiveProtocol3 = resource.mode ?? resource.protocol;
if (effectiveProtocol3 === "tcp" || effectiveProtocol3 === "udp") {
return resource.auth === undefined;
}
return true;
@@ -342,7 +423,8 @@ export const ResourceSchema = z
if (parts.includes("*", 1)) return false; // no further wildcards
if (parts.length < 3) return false; // need at least *.label.tld
const labelRegex = /^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$|^[a-zA-Z0-9]$/;
const labelRegex =
/^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$|^[a-zA-Z0-9]$/;
return parts.slice(1).every((label) => labelRegex.test(label));
},
{
@@ -350,22 +432,46 @@ export const ResourceSchema = z
message:
'Wildcard full-domain must have "*" as the leftmost label only, followed by at least two valid hostname labels (e.g. "*.example.com" or "*.level1.example.com"). Patterns like "*example.com" or "level2.*.example.com" are not supported.'
}
);
)
.refine(
(resource) => {
const effectiveMode = resource.mode ?? resource.protocol;
if (effectiveMode !== "tcp") {
return (
resource["proxy-protocol"] === undefined &&
resource["proxy-protocol-version"] === undefined
);
}
return true;
},
{
path: ["proxy-protocol"],
message:
"'proxy-protocol' and 'proxy-protocol-version' can only be set when mode is 'tcp'"
}
)
.transform((resource) => {
// Normalize: prefer mode, fall back to protocol for backwards compatibility
if (resource.mode === undefined && resource.protocol !== undefined) {
resource.mode = resource.protocol;
}
return resource;
});
export function isTargetsOnlyResource(resource: any): boolean {
return Object.keys(resource).length === 1 && resource.targets;
}
export const ClientResourceSchema = z
export const PrivateResourceSchema = z
.object({
name: z.string().min(1).max(255),
mode: z.enum(["host", "cidr", "http"]),
mode: z.enum(["host", "cidr", "http", "ssh"]),
site: z.string().optional(), // DEPRECATED IN FAVOR OF sites
sites: z.array(z.string()).optional().default([]),
// protocol: z.enum(["tcp", "udp"]).optional(),
// proxyPort: z.int().positive().optional(),
"destination-port": z.int().positive().optional(),
destination: z.string().min(1),
destination: z.string().min(1).optional(),
// enabled: z.boolean().default(true),
"tcp-ports": portRangeStringSchema.optional().default("*"),
"udp-ports": portRangeStringSchema.optional().default("*"),
@@ -388,11 +494,31 @@ export const ClientResourceSchema = z
error: "Admin role cannot be included in roles"
}),
users: z.array(z.string()).optional().default([]),
machines: z.array(z.string()).optional().default([])
machines: z.array(z.string()).optional().default([]),
"auth-daemon": AuthDaemonSchema.optional()
})
.refine(
(data) => {
// destination is optional only for ssh+native; required for everything else
const isNativeSSH =
data.mode === "ssh" &&
(data["auth-daemon"] === undefined ||
data["auth-daemon"].mode === "native");
if (!isNativeSSH && !data.destination) {
return false;
}
return true;
},
{
path: ["destination"],
message:
"destination is required unless mode is 'ssh' with auth-daemon mode 'native'"
}
)
.refine(
(data) => {
if (data.mode === "host") {
if (!data.destination) return true; // caught by the destination-required refine
// Check if it's a valid IP address using zod (v4 or v6)
const isValidIP = z
.union([z.ipv4(), z.ipv6()])
@@ -420,6 +546,7 @@ export const ClientResourceSchema = z
.refine(
(data) => {
if (data.mode === "cidr") {
if (!data.destination) return true; // caught by the destination-required refine
// Check if it's a valid CIDR (v4 or v6)
const isValidCIDR = z
.union([z.cidrv4(), z.cidrv6()])
@@ -431,25 +558,112 @@ export const ClientResourceSchema = z
{
message: "Destination must be a valid CIDR notation for cidr mode"
}
);
)
.refine(
(data) => {
if (data.mode !== "ssh") {
return true;
}
const authDaemonMode = data["auth-daemon"]?.mode;
if (authDaemonMode !== "native" && authDaemonMode !== "site") {
return true;
}
const uniqueSites = new Set<string>();
if (data.site) {
uniqueSites.add(data.site);
}
for (const site of data.sites) {
uniqueSites.add(site);
}
return uniqueSites.size <= 1;
},
{
path: ["sites"],
message:
"When mode is 'ssh' and auth-daemon mode is 'native' or 'site', only one site/target is allowed"
}
)
.transform((data) => {
if (
data.mode === "ssh" &&
data.destination !== undefined &&
data["destination-port"] === undefined
) {
data["destination-port"] = 22;
}
return data;
});
export const ResourcePolicyRuleSchema = RuleSchema;
export const ResourcePolicySchema = z.object({
name: z.string().min(1).max(255),
sso: z.boolean().optional().default(true),
"auto-login-idp": z.int().positive().optional().nullable(),
"sso-roles": z
.array(z.string())
.optional()
.default([])
.refine((roles) => !roles.includes("Admin"), {
error: "Admin role cannot be included in sso-roles"
}),
"sso-users": z.array(z.string()).optional().default([]),
password: z.string().min(4).max(100).optional().nullable(),
pincode: z
.string()
.regex(/^\d{6}$/)
.optional()
.nullable(),
"basic-auth": z
.object({
user: z.string().min(4).max(100),
password: z.string().min(4).max(100),
"extended-compatibility": z.boolean().default(true)
})
.optional()
.nullable(),
"email-whitelist-enabled": z.boolean().optional().default(false),
"whitelist-users": z
.array(
z.email().or(
z.string().regex(/^\*@[\w.-]+\.[a-zA-Z]{2,}$/, {
error: "Invalid email address. Wildcard (*) must be the entire local part."
})
)
)
.max(50)
.transform((v) => v.map((e) => e.toLowerCase()))
.optional()
.default([]),
"apply-rules": z.boolean().optional().default(false),
rules: z.array(ResourcePolicyRuleSchema).optional().default([])
});
export type ResourcePolicyData = z.infer<typeof ResourcePolicySchema>;
// Schema for the entire configuration object
export const ConfigSchema = z
.object({
"proxy-resources": z
.record(z.string(), ResourceSchema)
.record(z.string(), PublicResourceSchema)
.optional()
.prefault({}),
"public-resources": z
.record(z.string(), ResourceSchema)
.record(z.string(), PublicResourceSchema)
.optional()
.prefault({}),
"client-resources": z
.record(z.string(), ClientResourceSchema)
.record(z.string(), PrivateResourceSchema)
.optional()
.prefault({}),
"private-resources": z
.record(z.string(), ClientResourceSchema)
.record(z.string(), PrivateResourceSchema)
.optional()
.prefault({}),
"public-policies": z
.record(z.string(), ResourcePolicySchema)
.optional()
.prefault({}),
sites: z.record(z.string(), SiteSchema).optional().prefault({})
@@ -474,10 +688,17 @@ export const ConfigSchema = z
}
return data as {
"proxy-resources": Record<string, z.infer<typeof ResourceSchema>>;
"proxy-resources": Record<
string,
z.infer<typeof PublicResourceSchema>
>;
"client-resources": Record<
string,
z.infer<typeof ClientResourceSchema>
z.infer<typeof PrivateResourceSchema>
>;
"public-policies": Record<
string,
z.infer<typeof ResourcePolicySchema>
>;
sites: Record<string, z.infer<typeof SiteSchema>>;
};
@@ -616,5 +837,6 @@ export const ConfigSchema = z
// Type inference from the schema
export type Site = z.infer<typeof SiteSchema>;
export type Target = z.infer<typeof TargetSchema>;
export type Resource = z.infer<typeof ResourceSchema>;
export type Resource = z.infer<typeof PublicResourceSchema>;
export type Config = z.infer<typeof ConfigSchema>;
export type BlueprintResourcePolicy = z.infer<typeof ResourcePolicySchema>;
+413 -407
View File
@@ -6,6 +6,7 @@ import {
db,
olms,
orgs,
primaryDb,
roleClients,
roles,
Transaction,
@@ -23,422 +24,427 @@ import { rebuildClientAssociationsFromClient } from "./rebuildClientAssociations
import { OlmErrorCodes } from "@server/routers/olm/error";
import { tierMatrix } from "./billing/tierMatrix";
export async function calculateUserClientsForOrgs(
type ClientRow = typeof clients.$inferSelect;
function runQueuedClientAssociationRebuilds(
userId: string,
trx: Transaction | typeof db = db
): Promise<void> {
const execute = async (transaction: Transaction | typeof db) => {
const orgCache = new Map<string, typeof orgs.$inferSelect | null>();
const adminRoleCache = new Map<
string,
typeof roles.$inferSelect | null
>();
const exitNodesCache = new Map<
string,
Awaited<ReturnType<typeof listExitNodes>>
>();
const isOrgLicensedCache = new Map<string, boolean>();
const existingClientCache = new Map<
string,
typeof clients.$inferSelect | null
>();
const roleClientAccessCache = new Map<string, boolean>();
const userClientAccessCache = new Map<string, boolean>();
queuedClients: ClientRow[]
) {
if (queuedClients.length === 0) {
return;
}
const getOrgOlmKey = (orgId: string, olmId: string) =>
`${orgId}:${olmId}`;
const getRoleClientKey = (roleId: number, clientId: number) =>
`${roleId}:${clientId}`;
const getUserClientKey = (cachedUserId: string, clientId: number) =>
`${cachedUserId}:${clientId}`;
const uniqueClientsById = new Map<number, ClientRow>();
for (const client of queuedClients) {
uniqueClientsById.set(client.clientId, client);
}
const getOrg = async (orgId: string) => {
if (orgCache.has(orgId)) {
return orgCache.get(orgId) ?? null;
}
const [org] = await transaction
.select()
.from(orgs)
.where(eq(orgs.orgId, orgId));
orgCache.set(orgId, org ?? null);
return org ?? null;
};
const getAdminRole = async (orgId: string) => {
if (adminRoleCache.has(orgId)) {
return adminRoleCache.get(orgId) ?? null;
}
const [adminRole] = await transaction
.select()
.from(roles)
.where(and(eq(roles.isAdmin, true), eq(roles.orgId, orgId)))
.limit(1);
adminRoleCache.set(orgId, adminRole ?? null);
return adminRole ?? null;
};
const getExitNodes = async (orgId: string) => {
if (exitNodesCache.has(orgId)) {
return exitNodesCache.get(orgId)!;
}
const exitNodes = await listExitNodes(orgId);
exitNodesCache.set(orgId, exitNodes);
return exitNodes;
};
const getIsOrgLicensed = async (orgId: string) => {
if (isOrgLicensedCache.has(orgId)) {
return isOrgLicensedCache.get(orgId)!;
}
const isOrgLicensed = await isLicensedOrSubscribed(
orgId,
tierMatrix.deviceApprovals
for (const client of uniqueClientsById.values()) {
rebuildClientAssociationsFromClient(client).catch((error) => {
logger.error(
`Error rebuilding client associations for client ${client.clientId} (user ${userId}): ${String(
error
)}`
);
isOrgLicensedCache.set(orgId, isOrgLicensed);
return isOrgLicensed;
};
const getExistingClient = async (orgId: string, olmId: string) => {
const key = getOrgOlmKey(orgId, olmId);
if (existingClientCache.has(key)) {
return existingClientCache.get(key) ?? null;
}
const [existingClient] = await transaction
.select()
.from(clients)
.where(
and(
eq(clients.userId, userId),
eq(clients.orgId, orgId),
eq(clients.olmId, olmId)
)
)
.limit(1);
existingClientCache.set(key, existingClient ?? null);
return existingClient ?? null;
};
const hasRoleClientAccess = async (
roleId: number,
clientId: number
) => {
const key = getRoleClientKey(roleId, clientId);
if (roleClientAccessCache.has(key)) {
return roleClientAccessCache.get(key)!;
}
const [existingRoleClient] = await transaction
.select()
.from(roleClients)
.where(
and(
eq(roleClients.roleId, roleId),
eq(roleClients.clientId, clientId)
)
)
.limit(1);
const hasAccess = Boolean(existingRoleClient);
roleClientAccessCache.set(key, hasAccess);
return hasAccess;
};
const hasUserClientAccess = async (
cachedUserId: string,
clientId: number
) => {
const key = getUserClientKey(cachedUserId, clientId);
if (userClientAccessCache.has(key)) {
return userClientAccessCache.get(key)!;
}
const [existingUserClient] = await transaction
.select()
.from(userClients)
.where(
and(
eq(userClients.userId, cachedUserId),
eq(userClients.clientId, clientId)
)
)
.limit(1);
const hasAccess = Boolean(existingUserClient);
userClientAccessCache.set(key, hasAccess);
return hasAccess;
};
// Get all OLMs for this user
const userOlms = await transaction
.select()
.from(olms)
.where(eq(olms.userId, userId));
if (userOlms.length === 0) {
// No OLMs for this user, but we should still clean up any orphaned clients
await cleanupOrphanedClients(userId, transaction);
return;
}
// Get all user orgs with all roles (for org list and role-based logic)
const userOrgRoleRows = await transaction
.select()
.from(userOrgs)
.innerJoin(
userOrgRoles,
and(
eq(userOrgs.userId, userOrgRoles.userId),
eq(userOrgs.orgId, userOrgRoles.orgId)
)
)
.innerJoin(roles, eq(userOrgRoles.roleId, roles.roleId))
.where(eq(userOrgs.userId, userId));
const userOrgIds = [
...new Set(userOrgRoleRows.map((r) => r.userOrgs.orgId))
];
const orgIdToRoleRows = new Map<
string,
(typeof userOrgRoleRows)[0][]
>();
for (const r of userOrgRoleRows) {
const list = orgIdToRoleRows.get(r.userOrgs.orgId) ?? [];
list.push(r);
orgIdToRoleRows.set(r.userOrgs.orgId, list);
}
const orgRequiresDeviceApprovalRole = new Map<string, boolean>();
for (const [orgId, roleRowsForOrg] of orgIdToRoleRows.entries()) {
orgRequiresDeviceApprovalRole.set(
orgId,
roleRowsForOrg.some((r) => r.roles.requireDeviceApproval)
);
}
// For each OLM, ensure there's a client in each org the user is in
for (const olm of userOlms) {
for (const orgId of orgIdToRoleRows.keys()) {
const roleRowsForOrg = orgIdToRoleRows.get(orgId)!;
const userOrg = roleRowsForOrg[0].userOrgs;
const org = await getOrg(orgId);
if (!org) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): org not found`
);
continue;
}
if (!org.subnet) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): org has no subnet configured`
);
continue;
}
// Get admin role for this org (needed for access grants)
const adminRole = await getAdminRole(orgId);
if (!adminRole) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no admin role found`
);
continue;
}
// Check if a client already exists for this OLM+user+org combination
const existingClient = await getExistingClient(
orgId,
olm.olmId
);
if (existingClient) {
// Ensure admin role has access to the client
const hasRoleAccess = await hasRoleClientAccess(
adminRole.roleId,
existingClient.clientId
);
if (!hasRoleAccess) {
await transaction.insert(roleClients).values({
roleId: adminRole.roleId,
clientId: existingClient.clientId
});
roleClientAccessCache.set(
getRoleClientKey(
adminRole.roleId,
existingClient.clientId
),
true
);
logger.debug(
`Granted admin role access to existing client ${existingClient.clientId} for OLM ${olm.olmId} in org ${orgId} (user ${userId})`
);
}
// Ensure user has access to the client
const hasUserAccess = await hasUserClientAccess(
userId,
existingClient.clientId
);
if (!hasUserAccess) {
await transaction.insert(userClients).values({
userId,
clientId: existingClient.clientId
});
userClientAccessCache.set(
getUserClientKey(userId, existingClient.clientId),
true
);
logger.debug(
`Granted user access to existing client ${existingClient.clientId} for OLM ${olm.olmId} in org ${orgId} (user ${userId})`
);
}
logger.debug(
`Client already exists for OLM ${olm.olmId} in org ${orgId} (user ${userId}), skipping creation`
);
continue;
}
// Get exit nodes for this org
const exitNodesList = await getExitNodes(orgId);
if (exitNodesList.length === 0) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no exit nodes found`
);
continue;
}
const randomExitNode =
exitNodesList[
Math.floor(Math.random() * exitNodesList.length)
];
// Get next available subnet
const newSubnet = await getNextAvailableClientSubnet(
orgId,
transaction
);
if (!newSubnet) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no available subnet found`
);
continue;
}
const subnet = newSubnet.split("/")[0];
const updatedSubnet = `${subnet}/${org.subnet.split("/")[1]}`;
const niceId = await getUniqueClientName(orgId);
const isOrgLicensed = await getIsOrgLicensed(userOrg.orgId);
const requireApproval =
build !== "oss" &&
isOrgLicensed &&
orgRequiresDeviceApprovalRole.get(orgId) === true;
const newClientData: InferInsertModel<typeof clients> = {
userId,
orgId: userOrg.orgId,
exitNodeId: randomExitNode.exitNodeId,
name: olm.name || "User Client",
subnet: updatedSubnet,
olmId: olm.olmId,
type: "olm",
niceId,
approvalState: requireApproval ? "pending" : null
};
// Create the client
const [newClient] = await transaction
.insert(clients)
.values(newClientData)
.returning();
existingClientCache.set(
getOrgOlmKey(orgId, olm.olmId),
newClient
);
// create approval request
if (requireApproval) {
await transaction
.insert(approvals)
.values({
timestamp: Math.floor(new Date().getTime() / 1000),
orgId: userOrg.orgId,
clientId: newClient.clientId,
userId,
type: "user_device"
})
.returning();
}
await rebuildClientAssociationsFromClient(
newClient,
transaction
);
// Grant admin role access to the client
await transaction.insert(roleClients).values({
roleId: adminRole.roleId,
clientId: newClient.clientId
});
roleClientAccessCache.set(
getRoleClientKey(adminRole.roleId, newClient.clientId),
true
);
// Grant user access to the client
await transaction.insert(userClients).values({
userId,
clientId: newClient.clientId
});
userClientAccessCache.set(
getUserClientKey(userId, newClient.clientId),
true
);
logger.debug(
`Created client for OLM ${olm.olmId} in org ${orgId} (user ${userId}) with access granted to admin role and user`
);
}
}
// Clean up clients in orgs the user is no longer in
await cleanupOrphanedClients(userId, transaction, userOrgIds);
};
if (trx) {
// Use provided transaction
await execute(trx);
} else {
// Create new transaction
await db.transaction(async (transaction) => {
await execute(transaction);
});
}
logger.debug(
`Queued association rebuild completed for ${uniqueClientsById.size} client(s) (user ${userId})`
);
}
export async function calculateUserClientsForOrgs(
userId: string
): Promise<void> {
const trx = primaryDb;
const queuedAssociationRebuilds: ClientRow[] = [];
const orgCache = new Map<string, typeof orgs.$inferSelect | null>();
const adminRoleCache = new Map<string, typeof roles.$inferSelect | null>();
const exitNodesCache = new Map<
string,
Awaited<ReturnType<typeof listExitNodes>>
>();
const isOrgLicensedCache = new Map<string, boolean>();
const existingClientCache = new Map<
string,
typeof clients.$inferSelect | null
>();
const roleClientAccessCache = new Map<string, boolean>();
const userClientAccessCache = new Map<string, boolean>();
const getOrgOlmKey = (orgId: string, olmId: string) => `${orgId}:${olmId}`;
const getRoleClientKey = (roleId: number, clientId: number) =>
`${roleId}:${clientId}`;
const getUserClientKey = (cachedUserId: string, clientId: number) =>
`${cachedUserId}:${clientId}`;
const getOrg = async (orgId: string) => {
if (orgCache.has(orgId)) {
return orgCache.get(orgId) ?? null;
}
const [org] = await trx
.select()
.from(orgs)
.where(eq(orgs.orgId, orgId));
orgCache.set(orgId, org ?? null);
return org ?? null;
};
const getAdminRole = async (orgId: string) => {
if (adminRoleCache.has(orgId)) {
return adminRoleCache.get(orgId) ?? null;
}
const [adminRole] = await trx
.select()
.from(roles)
.where(and(eq(roles.isAdmin, true), eq(roles.orgId, orgId)))
.limit(1);
adminRoleCache.set(orgId, adminRole ?? null);
return adminRole ?? null;
};
const getExitNodes = async (orgId: string) => {
if (exitNodesCache.has(orgId)) {
return exitNodesCache.get(orgId)!;
}
const exitNodes = await listExitNodes(orgId);
exitNodesCache.set(orgId, exitNodes);
return exitNodes;
};
const getIsOrgLicensed = async (orgId: string) => {
if (isOrgLicensedCache.has(orgId)) {
return isOrgLicensedCache.get(orgId)!;
}
const isOrgLicensed = await isLicensedOrSubscribed(
orgId,
tierMatrix.deviceApprovals
);
isOrgLicensedCache.set(orgId, isOrgLicensed);
return isOrgLicensed;
};
const getExistingClient = async (orgId: string, olmId: string) => {
const key = getOrgOlmKey(orgId, olmId);
if (existingClientCache.has(key)) {
return existingClientCache.get(key) ?? null;
}
const [existingClient] = await trx
.select()
.from(clients)
.where(
and(
eq(clients.userId, userId),
eq(clients.orgId, orgId),
eq(clients.olmId, olmId)
)
)
.limit(1);
existingClientCache.set(key, existingClient ?? null);
return existingClient ?? null;
};
const hasRoleClientAccess = async (roleId: number, clientId: number) => {
const key = getRoleClientKey(roleId, clientId);
if (roleClientAccessCache.has(key)) {
return roleClientAccessCache.get(key)!;
}
const [existingRoleClient] = await trx
.select()
.from(roleClients)
.where(
and(
eq(roleClients.roleId, roleId),
eq(roleClients.clientId, clientId)
)
)
.limit(1);
const hasAccess = Boolean(existingRoleClient);
roleClientAccessCache.set(key, hasAccess);
return hasAccess;
};
const hasUserClientAccess = async (
cachedUserId: string,
clientId: number
) => {
const key = getUserClientKey(cachedUserId, clientId);
if (userClientAccessCache.has(key)) {
return userClientAccessCache.get(key)!;
}
const [existingUserClient] = await trx
.select()
.from(userClients)
.where(
and(
eq(userClients.userId, cachedUserId),
eq(userClients.clientId, clientId)
)
)
.limit(1);
const hasAccess = Boolean(existingUserClient);
userClientAccessCache.set(key, hasAccess);
return hasAccess;
};
// Get all OLMs for this user
const userOlms = await trx
.select()
.from(olms)
.where(eq(olms.userId, userId));
if (userOlms.length === 0) {
// No OLMs for this user, but we should still clean up any orphaned clients
await cleanupOrphanedClients(
userId,
trx,
[],
queuedAssociationRebuilds
);
return;
}
// Get all user orgs with all roles (for org list and role-based logic)
const userOrgRoleRows = await trx
.select()
.from(userOrgs)
.innerJoin(
userOrgRoles,
and(
eq(userOrgs.userId, userOrgRoles.userId),
eq(userOrgs.orgId, userOrgRoles.orgId)
)
)
.innerJoin(roles, eq(userOrgRoles.roleId, roles.roleId))
.where(eq(userOrgs.userId, userId));
const userOrgIds = [
...new Set(userOrgRoleRows.map((r) => r.userOrgs.orgId))
];
const orgIdToRoleRows = new Map<string, (typeof userOrgRoleRows)[0][]>();
for (const r of userOrgRoleRows) {
const list = orgIdToRoleRows.get(r.userOrgs.orgId) ?? [];
list.push(r);
orgIdToRoleRows.set(r.userOrgs.orgId, list);
}
const orgRequiresDeviceApprovalRole = new Map<string, boolean>();
for (const [orgId, roleRowsForOrg] of orgIdToRoleRows.entries()) {
orgRequiresDeviceApprovalRole.set(
orgId,
roleRowsForOrg.some((r) => r.roles.requireDeviceApproval)
);
}
// For each OLM, ensure there's a client in each org the user is in
for (const olm of userOlms) {
for (const orgId of orgIdToRoleRows.keys()) {
const roleRowsForOrg = orgIdToRoleRows.get(orgId)!;
const userOrg = roleRowsForOrg[0].userOrgs;
const org = await getOrg(orgId);
if (!org) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): org not found`
);
continue;
}
if (!org.subnet) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): org has no subnet configured`
);
continue;
}
// Get admin role for this org (needed for access grants)
const adminRole = await getAdminRole(orgId);
if (!adminRole) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no admin role found`
);
continue;
}
// Check if a client already exists for this OLM+user+org combination
const existingClient = await getExistingClient(orgId, olm.olmId);
if (existingClient) {
// Ensure admin role has access to the client
const hasRoleAccess = await hasRoleClientAccess(
adminRole.roleId,
existingClient.clientId
);
if (!hasRoleAccess) {
await trx.insert(roleClients).values({
roleId: adminRole.roleId,
clientId: existingClient.clientId
});
roleClientAccessCache.set(
getRoleClientKey(
adminRole.roleId,
existingClient.clientId
),
true
);
logger.debug(
`Granted admin role access to existing client ${existingClient.clientId} for OLM ${olm.olmId} in org ${orgId} (user ${userId})`
);
}
// Ensure user has access to the client
const hasUserAccess = await hasUserClientAccess(
userId,
existingClient.clientId
);
if (!hasUserAccess) {
await trx.insert(userClients).values({
userId,
clientId: existingClient.clientId
});
userClientAccessCache.set(
getUserClientKey(userId, existingClient.clientId),
true
);
logger.debug(
`Granted user access to existing client ${existingClient.clientId} for OLM ${olm.olmId} in org ${orgId} (user ${userId})`
);
}
logger.debug(
`Client already exists for OLM ${olm.olmId} in org ${orgId} (user ${userId}), skipping creation`
);
continue;
}
// Get exit nodes for this org
const exitNodesList = await getExitNodes(orgId);
if (exitNodesList.length === 0) {
logger.warn(
`Skipping org ${orgId} for OLM ${olm.olmId} (user ${userId}): no exit nodes found`
);
continue;
}
const randomExitNode =
exitNodesList[Math.floor(Math.random() * exitNodesList.length)];
// Get next available subnet
const { value: newSubnet, release: releaseSubnetLock } =
await getNextAvailableClientSubnet(orgId, trx);
const subnet = newSubnet.split("/")[0];
const updatedSubnet = `${subnet}/${org.subnet.split("/")[1]}`;
const niceId = await getUniqueClientName(orgId);
const isOrgLicensed = await getIsOrgLicensed(userOrg.orgId);
const requireApproval =
build !== "oss" &&
isOrgLicensed &&
orgRequiresDeviceApprovalRole.get(orgId) === true;
const newClientData: InferInsertModel<typeof clients> = {
userId,
orgId: userOrg.orgId,
exitNodeId: randomExitNode.exitNodeId,
name: olm.name || "User Client",
subnet: updatedSubnet,
olmId: olm.olmId,
type: "olm",
niceId,
approvalState: requireApproval ? "pending" : null
};
// Create the client
const [newClient] = await trx
.insert(clients)
.values(newClientData)
.returning();
await releaseSubnetLock();
existingClientCache.set(getOrgOlmKey(orgId, olm.olmId), newClient);
// create approval request
if (requireApproval) {
await trx
.insert(approvals)
.values({
timestamp: Math.floor(new Date().getTime() / 1000),
orgId: userOrg.orgId,
clientId: newClient.clientId,
userId,
type: "user_device"
})
.returning();
}
queuedAssociationRebuilds.push(newClient);
// Grant admin role access to the client
await trx.insert(roleClients).values({
roleId: adminRole.roleId,
clientId: newClient.clientId
});
roleClientAccessCache.set(
getRoleClientKey(adminRole.roleId, newClient.clientId),
true
);
// Grant user access to the client
await trx.insert(userClients).values({
userId,
clientId: newClient.clientId
});
userClientAccessCache.set(
getUserClientKey(userId, newClient.clientId),
true
);
logger.debug(
`Created client for OLM ${olm.olmId} in org ${orgId} (user ${userId}) with access granted to admin role and user`
);
}
}
// Clean up clients in orgs the user is no longer in
await cleanupOrphanedClients(
userId,
trx,
userOrgIds,
queuedAssociationRebuilds
);
runQueuedClientAssociationRebuilds(userId, queuedAssociationRebuilds);
}
async function cleanupOrphanedClients(
userId: string,
trx: Transaction | typeof db,
userOrgIds: string[] = []
userOrgIds: string[] = [],
queuedAssociationRebuilds: ClientRow[] = []
): Promise<void> {
// Find all OLM clients for this user that should be deleted
// If userOrgIds is empty, delete all OLM clients (user has no orgs)
@@ -468,9 +474,9 @@ async function cleanupOrphanedClients(
)
.returning();
// Rebuild associations for each deleted client to clean up related data
// Queue deleted clients for post-trx association cleanup.
for (const deletedClient of deletedClients) {
await rebuildClientAssociationsFromClient(deletedClient, trx);
queuedAssociationRebuilds.push(deletedClient);
if (deletedClient.olmId) {
await sendTerminateClient(
+1 -1
View File
@@ -2,7 +2,7 @@ import path from "path";
import { fileURLToPath } from "url";
// This is a placeholder value replaced by the build process
export const APP_VERSION = "1.18.4";
export const APP_VERSION = "1.19.0";
export const __FILENAME = fileURLToPath(import.meta.url);
export const __DIRNAME = path.dirname(__FILENAME);
+83
View File
@@ -0,0 +1,83 @@
import logger from "@server/logger";
const MAX_RETRIES = 5;
const BASE_DELAY_MS = 50;
/**
* Detect transient errors that are safe to retry (connection drops, deadlocks,
* serialization failures). PostgreSQL deadlocks (40P01) are always safe to
* retry: the database guarantees exactly one winner per deadlock pair, so the
* loser just needs to try again.
*/
export function isTransientError(error: any): boolean {
if (!error) return false;
const message = (error.message || "").toLowerCase();
const causeMessage = (error.cause?.message || "").toLowerCase();
const code = error.code || error.cause?.code || "";
// Connection timeout / terminated
if (
message.includes("connection timeout") ||
message.includes("connection terminated") ||
message.includes("timeout exceeded when trying to connect") ||
causeMessage.includes("connection terminated unexpectedly") ||
causeMessage.includes("connection timeout")
) {
return true;
}
// PostgreSQL deadlock detected - always safe to retry (one winner guaranteed)
if (code === "40P01" || message.includes("deadlock")) {
return true;
}
// PostgreSQL serialization failure
if (code === "40001") {
return true;
}
// ECONNRESET, ECONNREFUSED, EPIPE, ETIMEDOUT
if (
code === "ECONNRESET" ||
code === "ECONNREFUSED" ||
code === "EPIPE" ||
code === "ETIMEDOUT"
) {
return true;
}
return false;
}
/**
* Simple retry wrapper with exponential backoff for transient errors
* (deadlocks, connection timeouts, unexpected disconnects).
*/
export async function withRetry<T>(
operation: () => Promise<T>,
context: string,
maxRetries: number = MAX_RETRIES,
baseDelayMs: number = BASE_DELAY_MS
): Promise<T> {
let attempt = 0;
while (true) {
try {
return await operation();
} catch (error: any) {
if (isTransientError(error) && attempt < maxRetries) {
attempt++;
const baseDelay = Math.pow(2, attempt - 1) * baseDelayMs;
const jitter = Math.random() * baseDelay;
const delay = baseDelay + jitter;
logger.warn(
`Transient DB error in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
{ code: error?.code ?? error?.cause?.code }
);
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
throw error;
}
}
}
+9 -7
View File
@@ -24,7 +24,7 @@ import { deletePeer } from "@server/routers/gerbil/peers";
import { OlmErrorCodes } from "@server/routers/olm/error";
import { sendTerminateClient } from "@server/routers/client/terminate";
import { usageService } from "./billing/usageService";
import { FeatureId } from "./billing";
import { LimitId } from "./billing";
export type DeleteOrgByIdResult = {
deletedNewtIds: string[];
@@ -140,7 +140,9 @@ export async function deleteOrgById(
.select({ count: count() })
.from(orgDomains)
.where(eq(orgDomains.domainId, domainId));
logger.info(`Found ${orgCount.count} orgs using domain ${domainId}`);
logger.info(
`Found ${orgCount.count} orgs using domain ${domainId}`
);
if (orgCount.count === 1) {
domainIdsToDelete.push(domainId);
}
@@ -152,7 +154,7 @@ export async function deleteOrgById(
.where(inArray(domains.domainId, domainIdsToDelete));
}
await usageService.add(orgId, FeatureId.ORGINIZATIONS, -1, trx); // here we are decreasing the org count BEFORE deleting the org because we need to still be able to get the org to get the billing org inside of here
await usageService.add(orgId, LimitId.ORGANIZATIONS, -1, trx); // here we are decreasing the org count BEFORE deleting the org because we need to still be able to get the org to get the billing org inside of here
await trx.delete(orgs).where(eq(orgs.orgId, orgId));
@@ -199,22 +201,22 @@ export async function deleteOrgById(
if (org.billingOrgId) {
usageService.updateCount(
org.billingOrgId,
FeatureId.DOMAINS,
LimitId.DOMAINS,
domainCount ?? 0
);
usageService.updateCount(
org.billingOrgId,
FeatureId.SITES,
LimitId.SITES,
siteCount ?? 0
);
usageService.updateCount(
org.billingOrgId,
FeatureId.USERS,
LimitId.USERS,
userCount ?? 0
);
usageService.updateCount(
org.billingOrgId,
FeatureId.REMOTE_EXIT_NODES,
LimitId.REMOTE_EXIT_NODES,
remoteExitNodeCount ?? 0
);
}
+144
View File
@@ -0,0 +1,144 @@
import { eq, inArray } from "drizzle-orm";
import {
db,
newts,
resourcePolicies,
resources,
sites,
targetHealthCheck,
targets,
type Resource,
type Target,
type TargetHealthCheck,
type Transaction
} from "@server/db";
import logger from "@server/logger";
import { removeTargets } from "@server/routers/newt/targets";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
export type DeleteResourceResult = {
deletedResource: Resource;
targetsToBeRemoved: Target[];
healthChecksToBeRemoved: TargetHealthCheck[];
};
export async function performDeleteResources(
resourceIds: number[],
trx: Transaction | typeof db = db
): Promise<DeleteResourceResult[]> {
if (resourceIds.length === 0) {
return [];
}
const targetsToBeRemoved = await trx
.select()
.from(targets)
.where(inArray(targets.resourceId, resourceIds));
const targetIds = targetsToBeRemoved.map((t) => t.targetId);
const healthChecksToBeRemoved =
targetIds.length > 0
? await trx
.select()
.from(targetHealthCheck)
.where(inArray(targetHealthCheck.targetId, targetIds))
: [];
const deletedResources = await trx
.delete(resources)
.where(inArray(resources.resourceId, resourceIds))
.returning();
const policyIds = deletedResources
.map((resource) => resource.defaultResourcePolicyId)
.filter((id): id is number => id != null);
if (policyIds.length > 0) {
await trx
.delete(resourcePolicies)
.where(inArray(resourcePolicies.resourcePolicyId, policyIds));
}
if (deletedResources.length > 0) {
logger.debug(`Deleted ${deletedResources.length} resources`);
}
const targetsByResourceId = new Map<number, Target[]>();
for (const target of targetsToBeRemoved) {
const existing = targetsByResourceId.get(target.resourceId) ?? [];
existing.push(target);
targetsByResourceId.set(target.resourceId, existing);
}
const targetIdToResourceId = new Map(
targetsToBeRemoved.map((target) => [target.targetId, target.resourceId])
);
const healthChecksByResourceId = new Map<number, TargetHealthCheck[]>();
for (const healthCheck of healthChecksToBeRemoved) {
const resourceId = targetIdToResourceId.get(healthCheck.targetId!);
if (resourceId == null) {
continue;
}
const existing = healthChecksByResourceId.get(resourceId) ?? [];
existing.push(healthCheck);
healthChecksByResourceId.set(resourceId, existing);
}
return deletedResources.map((deletedResource) => ({
deletedResource,
targetsToBeRemoved:
targetsByResourceId.get(deletedResource.resourceId) ?? [],
healthChecksToBeRemoved:
healthChecksByResourceId.get(deletedResource.resourceId) ?? []
}));
}
export async function performDeleteResource(
resourceId: number,
trx: Transaction | typeof db = db
): Promise<DeleteResourceResult | null> {
const [result] = await performDeleteResources([resourceId], trx);
return result ?? null;
}
export async function runResourceDeleteSideEffects(
result: DeleteResourceResult
): Promise<void> {
const { deletedResource, targetsToBeRemoved, healthChecksToBeRemoved } =
result;
for (const target of targetsToBeRemoved) {
const [site] = await db
.select()
.from(sites)
.where(eq(sites.siteId, target.siteId))
.limit(1);
if (!site) {
throw createHttpError(
HttpCode.NOT_FOUND,
`Site with ID ${target.siteId} not found`
);
}
if (site.pubKey && site.type === "newt") {
const [newt] = await db
.select()
.from(newts)
.where(eq(newts.siteId, site.siteId))
.limit(1);
if (newt) {
await removeTargets(
newt.newtId,
[],
healthChecksToBeRemoved,
deletedResource.mode === "udp" ? "udp" : "tcp",
newt.version
);
}
}
}
}
+126
View File
@@ -0,0 +1,126 @@
import { and, eq, sql } from "drizzle-orm";
import {
db,
siteNetworks,
siteResources,
targets,
type SiteResource,
type Transaction
} from "@server/db";
import {
performDeleteResources,
runResourceDeleteSideEffects,
type DeleteResourceResult
} from "@server/lib/deleteResource";
import {
performDeleteSiteResources,
runSiteResourceDeleteSideEffects
} from "@server/lib/deleteSiteResource";
import logger from "@server/logger";
export const MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE = 250;
export type DeleteSiteAssociatedResourcesSideEffects = {
resources: DeleteResourceResult[];
siteResources: SiteResource[];
};
export async function getResourceIdsForSite(
siteId: number,
trx: Transaction | typeof db = db
): Promise<number[]> {
const rows = await trx
.selectDistinct({ resourceId: targets.resourceId })
.from(targets)
.where(eq(targets.siteId, siteId));
return rows.map((row) => row.resourceId);
}
export async function getSiteResourceIdsForSite(
siteId: number,
orgId: string,
trx: Transaction | typeof db = db
): Promise<number[]> {
const rows = await trx
.selectDistinct({ siteResourceId: siteResources.siteResourceId })
.from(siteNetworks)
.innerJoin(
siteResources,
eq(siteResources.networkId, siteNetworks.networkId)
)
.where(
and(eq(siteNetworks.siteId, siteId), eq(siteResources.orgId, orgId))
);
return rows.map((row) => row.siteResourceId);
}
export async function getAssociatedResourceCountForSite(
siteId: number,
orgId: string,
trx: Transaction | typeof db = db
): Promise<number> {
const [publicCountResult, privateCountResult] = await Promise.all([
trx
.select({
count: sql<number>`count(distinct ${targets.resourceId})`
})
.from(targets)
.where(eq(targets.siteId, siteId)),
trx
.select({
count: sql<number>`count(distinct ${siteResources.siteResourceId})`
})
.from(siteNetworks)
.innerJoin(
siteResources,
eq(siteResources.networkId, siteNetworks.networkId)
)
.where(
and(
eq(siteNetworks.siteId, siteId),
eq(siteResources.orgId, orgId)
)
)
]);
return (
Number(publicCountResult[0]?.count ?? 0) +
Number(privateCountResult[0]?.count ?? 0)
);
}
export function exceedsSiteAssociatedResourceDeleteLimit(
resourceCount: number
): boolean {
return resourceCount > MAX_SITE_ASSOCIATED_RESOURCES_FOR_BULK_DELETE;
}
export async function deleteAssociatedResourcesForSite(
siteId: number,
orgId: string,
trx: Transaction | typeof db = db
): Promise<DeleteSiteAssociatedResourcesSideEffects> {
const resourceIds = await getResourceIdsForSite(siteId, trx);
const siteResourceIds = await getSiteResourceIdsForSite(siteId, orgId, trx);
const [resources, siteResourcesDeleted] = await Promise.all([
performDeleteResources(resourceIds, trx),
performDeleteSiteResources(siteResourceIds, trx)
]);
return { resources, siteResources: siteResourcesDeleted };
}
export async function runDeleteSiteAssociatedResourcesSideEffects(
sideEffects: DeleteSiteAssociatedResourcesSideEffects
): Promise<void> {
for (const result of sideEffects.resources) {
await runResourceDeleteSideEffects(result);
}
for (const removed of sideEffects.siteResources) {
runSiteResourceDeleteSideEffects(removed);
}
}
+53
View File
@@ -0,0 +1,53 @@
import { inArray } from "drizzle-orm";
import {
db,
siteResources,
type SiteResource,
type Transaction
} from "@server/db";
import logger from "@server/logger";
import { rebuildClientAssociationsFromSiteResource } from "@server/lib/rebuildClientAssociations";
export async function performDeleteSiteResources(
siteResourceIds: number[],
trx: Transaction | typeof db = db
): Promise<SiteResource[]> {
if (siteResourceIds.length === 0) {
return [];
}
const removedSiteResources = await trx
.delete(siteResources)
.where(inArray(siteResources.siteResourceId, siteResourceIds))
.returning();
if (removedSiteResources.length > 0) {
logger.debug(`Deleted ${removedSiteResources.length} site resources`);
}
return removedSiteResources;
}
export async function performDeleteSiteResource(
siteResourceId: number,
trx: Transaction | typeof db = db
): Promise<SiteResource | null> {
const [removedSiteResource] = await performDeleteSiteResources(
[siteResourceId],
trx
);
return removedSiteResource ?? null;
}
export function runSiteResourceDeleteSideEffects(
removedSiteResource: SiteResource
): void {
rebuildClientAssociationsFromSiteResource(removedSiteResource).catch(
(err) => {
logger.error(
`Error rebuilding client associations for site resource ${removedSiteResource.siteResourceId}:`,
err
);
}
);
}
+5 -1
View File
@@ -19,7 +19,11 @@ export async function verifyExitNodeOrgAccess(
export async function listExitNodes(
orgId: string,
filterOnline = false,
noCloud = false
noCloud = false,
// Accepted for parity with the enterprise implementation (used there for
// site-label filtering of remote exit nodes). The OSS build has no remote
// exit nodes, so it is unused here.
siteId?: number
) {
// TODO: pick which nodes to send and ping better than just all of them that are not remote
const allExitNodes = await db
+49 -24
View File
@@ -1,30 +1,55 @@
import { db, exitNodes } from "@server/db";
import { db, exitNodes, Transaction } from "@server/db";
import config from "@server/lib/config";
import { findNextAvailableCidr } from "@server/lib/ip";
import { lockManager } from "#dynamic/lib/lock";
export async function getNextAvailableSubnet(): Promise<string> {
// Get all existing subnets from routes table
const existingAddresses = await db
.select({
address: exitNodes.address
})
.from(exitNodes);
const addresses = existingAddresses.map((a) => a.address);
let subnet = findNextAvailableCidr(
addresses,
config.getRawConfig().gerbil.block_size,
config.getRawConfig().gerbil.subnet_group
);
if (!subnet) {
throw new Error("No available subnets remaining in space");
/**
* Reserves the next available exit node subnet.
*
* Exit node subnets must never overlap with one another - regardless of
* which org(s) they belong to - since HA exit nodes can end up routing for
* the same org. This acquires a lock that the caller MUST release (via the
* returned `release`) only after the chosen address has been durably
* persisted (e.g. after the enclosing transaction commits), otherwise
* concurrent callers can race and pick the same subnet.
*/
export async function getNextAvailableSubnet(
trx: Transaction | typeof db = db
): Promise<{ value: string; release: () => Promise<void> }> {
const lockKey = "exit-node-subnet-allocation";
const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey, acquired);
// replace the last octet with 1
subnet =
subnet.split(".").slice(0, 3).join(".") +
".1" +
"/" +
subnet.split("/")[1];
return subnet;
try {
// Get all existing subnets from routes table
const existingAddresses = await trx
.select({
address: exitNodes.address
})
.from(exitNodes);
const addresses = existingAddresses.map((a) => a.address);
let subnet = findNextAvailableCidr(
addresses,
config.getRawConfig().gerbil.block_size,
config.getRawConfig().gerbil.subnet_group
);
if (!subnet) {
throw new Error("No available subnets remaining in space");
}
// replace the last octet with 1
subnet =
subnet.split(".").slice(0, 3).join(".") +
".1" +
"/" +
subnet.split("/")[1];
return { value: subnet, release };
} catch (e) {
await release();
throw e;
}
}
+161 -123
View File
@@ -327,127 +327,145 @@ export function doCidrsOverlap(cidr1: string, cidr2: string): boolean {
export async function getNextAvailableClientSubnet(
orgId: string,
transaction: Transaction | typeof db = db
): Promise<string> {
return await lockManager.withLock(
`client-subnet-allocation:${orgId}`,
async () => {
const [org] = await transaction
.select()
.from(orgs)
.where(eq(orgs.orgId, orgId));
): Promise<{ value: string; release: () => Promise<void> }> {
const lockKey = `client-subnet-allocation:${orgId}`;
const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey, acquired);
if (!org) {
throw new Error(`Organization with ID ${orgId} not found`);
}
try {
const [org] = await transaction
.select()
.from(orgs)
.where(eq(orgs.orgId, orgId));
if (!org.subnet) {
throw new Error(
`Organization with ID ${orgId} has no subnet defined`
);
}
const existingAddressesSites = await transaction
.select({
address: sites.address
})
.from(sites)
.where(and(isNotNull(sites.address), eq(sites.orgId, orgId)));
const existingAddressesClients = await transaction
.select({
address: clients.subnet
})
.from(clients)
.where(
and(isNotNull(clients.subnet), eq(clients.orgId, orgId))
);
const addresses = [
...existingAddressesSites.map(
(site) => `${site.address?.split("/")[0]}/32`
), // we are overriding the 32 so that we pick individual addresses in the subnet of the org for the site and the client even though they are stored with the /block_size of the org
...existingAddressesClients.map(
(client) => `${client.address.split("/")}/32`
)
].filter((address) => address !== null) as string[];
const subnet = findNextAvailableCidr(addresses, 32, org.subnet); // pick the sites address in the org
if (!subnet) {
throw new Error("No available subnets remaining in space");
}
return subnet;
if (!org) {
throw new Error(`Organization with ID ${orgId} not found`);
}
);
if (!org.subnet) {
throw new Error(
`Organization with ID ${orgId} has no subnet defined`
);
}
const existingAddressesSites = await transaction
.select({
address: sites.address
})
.from(sites)
.where(and(isNotNull(sites.address), eq(sites.orgId, orgId)));
const existingAddressesClients = await transaction
.select({
address: clients.subnet
})
.from(clients)
.where(and(isNotNull(clients.subnet), eq(clients.orgId, orgId)));
const addresses = [
...existingAddressesSites.map(
(site) => `${site.address?.split("/")[0]}/32`
), // we are overriding the 32 so that we pick individual addresses in the subnet of the org for the site and the client even though they are stored with the /block_size of the org
...existingAddressesClients.map(
(client) => `${client.address.split("/")[0]}/32`
)
].filter((address) => address !== null) as string[];
const subnet = findNextAvailableCidr(addresses, 32, org.subnet); // pick the sites address in the org
if (!subnet) {
throw new Error("No available subnets remaining in space");
}
return { value: subnet, release };
} catch (e) {
await release();
throw e;
}
}
export async function getNextAvailableAliasAddress(
orgId: string,
trx: Transaction | typeof db = db
): Promise<string> {
return await lockManager.withLock(
`alias-address-allocation:${orgId}`,
async () => {
const [org] = await trx
.select()
.from(orgs)
.where(eq(orgs.orgId, orgId));
): Promise<{ value: string; release: () => Promise<void> }> {
const lockKey = `alias-address-allocation:${orgId}`;
const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey, acquired);
if (!org) {
throw new Error(`Organization with ID ${orgId} not found`);
}
try {
const [org] = await trx
.select()
.from(orgs)
.where(eq(orgs.orgId, orgId));
if (!org.subnet) {
throw new Error(
`Organization with ID ${orgId} has no subnet defined`
);
}
if (!org.utilitySubnet) {
throw new Error(
`Organization with ID ${orgId} has no utility subnet defined`
);
}
const existingAddresses = await trx
.select({
aliasAddress: siteResources.aliasAddress
})
.from(siteResources)
.where(
and(
isNotNull(siteResources.aliasAddress),
eq(siteResources.orgId, orgId)
)
);
const addresses = [
...existingAddresses.map(
(site) => `${site.aliasAddress?.split("/")[0]}/32`
),
// reserve a /29 for the dns server and other stuff
`${org.utilitySubnet.split("/")[0]}/29`
].filter((address) => address !== null) as string[];
let subnet = findNextAvailableCidr(
addresses,
32,
org.utilitySubnet
);
if (!subnet) {
throw new Error("No available subnets remaining in space");
}
// remove the cidr
subnet = subnet.split("/")[0];
return subnet;
if (!org) {
throw new Error(`Organization with ID ${orgId} not found`);
}
);
if (!org.subnet) {
throw new Error(
`Organization with ID ${orgId} has no subnet defined`
);
}
if (!org.utilitySubnet) {
throw new Error(
`Organization with ID ${orgId} has no utility subnet defined`
);
}
const existingAddresses = await trx
.select({
aliasAddress: siteResources.aliasAddress
})
.from(siteResources)
.where(
and(
isNotNull(siteResources.aliasAddress),
eq(siteResources.orgId, orgId)
)
);
const addresses = [
...existingAddresses.map(
(site) => `${site.aliasAddress?.split("/")[0]}/32`
),
// reserve a /29 for the dns server and other stuff
`${org.utilitySubnet.split("/")[0]}/29`
].filter((address) => address !== null) as string[];
let subnet = findNextAvailableCidr(addresses, 32, org.utilitySubnet);
if (!subnet) {
throw new Error("No available subnets remaining in space");
}
// remove the cidr
subnet = subnet.split("/")[0];
return { value: subnet, release };
} catch (e) {
await release();
throw e;
}
}
export async function getNextAvailableOrgSubnet(): Promise<string> {
return await lockManager.withLock("org-subnet-allocation", async () => {
export async function getNextAvailableOrgSubnet(): Promise<{
value: string;
release: () => Promise<void>;
}> {
const lockKey = "org-subnet-allocation";
const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
if (!acquired) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
const release = () => lockManager.releaseLock(lockKey, acquired);
try {
const existingAddresses = await db
.select({
subnet: orgs.subnet
@@ -466,8 +484,11 @@ export async function getNextAvailableOrgSubnet(): Promise<string> {
throw new Error("No available subnets remaining in space");
}
return subnet;
});
return { value: subnet, release };
} catch (e) {
await release();
throw e;
}
}
export function generateRemoteSubnets(
@@ -475,13 +496,15 @@ export function generateRemoteSubnets(
): string[] {
const remoteSubnets = allSiteResources
.filter((sr) => {
if (!sr.destination) return false;
if (sr.mode === "cidr") {
// check if its a valid CIDR using zod
const cidrSchema = z.union([z.cidrv4(), z.cidrv6()]);
const parseResult = cidrSchema.safeParse(sr.destination);
return parseResult.success;
}
if (sr.mode === "host") {
if (sr.mode === "host" || sr.mode === "ssh") {
// check if its a valid IP using zod
const ipSchema = z.union([z.ipv4(), z.ipv6()]);
const parseResult = ipSchema.safeParse(sr.destination);
@@ -491,12 +514,12 @@ export function generateRemoteSubnets(
})
.map((sr) => {
if (sr.mode === "cidr") return sr.destination;
if (sr.mode === "host") {
if (sr.mode === "host" || sr.mode === "ssh") {
return `${sr.destination}/32`;
}
return ""; // This should never be reached due to filtering, but satisfies TypeScript
})
.filter((subnet) => subnet !== ""); // Remove empty strings just to be safe
.filter((subnet): subnet is string => subnet !== "" && subnet !== null); // Remove invalid values just to be safe
// remove duplicates
return Array.from(new Set(remoteSubnets));
}
@@ -508,7 +531,7 @@ export function generateAliasConfig(allSiteResources: SiteResource[]): Alias[] {
.filter(
(sr) =>
sr.aliasAddress &&
((sr.alias && sr.mode == "host") ||
((sr.alias && (sr.mode == "host" || sr.mode == "ssh")) ||
(sr.fullDomain && sr.mode == "http"))
)
.map((sr) => ({
@@ -554,6 +577,10 @@ export function generateSubnetProxyTargets(
continue;
}
if (!siteResource.destination) {
continue;
}
const clientPrefix = `${clientSite.subnet.split("/")[0]}/32`;
const portRange = [
...parsePortRangeString(siteResource.tcpPortRangeString, "tcp"),
@@ -561,7 +588,7 @@ export function generateSubnetProxyTargets(
];
const disableIcmp = siteResource.disableIcmp ?? false;
if (siteResource.mode == "host") {
if (siteResource.mode == "host" || siteResource.mode == "ssh") {
let destination = siteResource.destination;
// check if this is a valid ip
const ipSchema = z.union([z.ipv4(), z.ipv6()]);
@@ -581,7 +608,7 @@ export function generateSubnetProxyTargets(
targets.push({
sourcePrefix: clientPrefix,
destPrefix: `${siteResource.aliasAddress}/32`,
rewriteTo: destination,
rewriteTo: destination!,
portRange,
disableIcmp
});
@@ -589,7 +616,7 @@ export function generateSubnetProxyTargets(
} else if (siteResource.mode == "cidr") {
targets.push({
sourcePrefix: clientPrefix,
destPrefix: siteResource.destination,
destPrefix: siteResource.destination!,
portRange,
disableIcmp
});
@@ -642,7 +669,12 @@ export async function generateSubnetProxyTargetV2(
return;
}
let targets: SubnetProxyTargetV2[] = [];
if (!siteResource.destination) {
// ssh can have no destination
return;
}
const targets: SubnetProxyTargetV2[] = [];
const portRange = [
...parsePortRangeString(siteResource.tcpPortRangeString, "tcp"),
@@ -650,7 +682,7 @@ export async function generateSubnetProxyTargetV2(
];
const disableIcmp = siteResource.disableIcmp ?? false;
if (siteResource.mode == "host") {
if (siteResource.mode == "host" || siteResource.mode == "ssh") {
let destination = siteResource.destination;
// check if this is a valid ip
const ipSchema = z.union([z.ipv4(), z.ipv6()]);
@@ -671,7 +703,7 @@ export async function generateSubnetProxyTargetV2(
targets.push({
sourcePrefixes: [],
destPrefix: `${siteResource.aliasAddress}/32`,
rewriteTo: destination,
rewriteTo: destination!,
portRange,
disableIcmp,
resourceId: siteResource.siteResourceId
@@ -680,7 +712,7 @@ export async function generateSubnetProxyTargetV2(
} else if (siteResource.mode == "cidr") {
targets.push({
sourcePrefixes: [],
destPrefix: siteResource.destination,
destPrefix: siteResource.destination!,
portRange,
disableIcmp,
resourceId: siteResource.siteResourceId
@@ -738,7 +770,7 @@ export async function generateSubnetProxyTargetV2(
protocol: siteResource.ssl ? "https" : "http",
httpTargets: [
{
destAddr: siteResource.destination,
destAddr: siteResource.destination!,
destPort: siteResource.destinationPort,
scheme: siteResource.scheme
}
@@ -873,7 +905,13 @@ export const portRangeStringSchema = z
message:
'Port range must be "*" for all ports, or a comma-separated list of ports and ranges (e.g., "80,443,8000-9000"). Ports must be between 1 and 65535, and ranges must have start <= end.'
}
);
)
.openapi({
type: "string",
description:
'Port range string. Use "*" for all ports, a comma-separated list of ports, or ranges (e.g., "80,443,8000-9000"). Ports must be between 1 and 65535.',
example: "80,443,8000-9000"
});
/**
* Parses a port range string into an array of port range objects
+144 -24
View File
@@ -1,28 +1,87 @@
import { randomUUID } from "crypto";
const instanceId = `local-${Math.random().toString(36).slice(2)}-${Date.now()}`;
type LocalLockRecord = {
owner: string;
expiresAt: number;
};
const localLocks = new Map<string, LocalLockRecord>();
export class LockManager {
/**
* Acquire a distributed lock using Redis SET with NX and PX options
* @param lockKey - Unique identifier for the lock
* @param ttlMs - Time to live in milliseconds
* @returns Promise<boolean> - true if lock acquired, false otherwise
*/
async acquireLock(
lockKey: string,
ttlMs: number = 30000
): Promise<boolean> {
return true;
private clearExpiredLocalLock(lockKey: string): void {
const current = localLocks.get(lockKey);
if (current && current.expiresAt <= Date.now()) {
localLocks.delete(lockKey);
}
}
/**
* Release a lock using Lua script to ensure atomicity
* Acquire a local in-process lock using an optimistic Map-based check.
* @param lockKey - Unique identifier for the lock
* @param ttlMs - Time to live in milliseconds
* @returns Promise<string | null> - a token identifying this specific acquisition
* (truthy) on success, or null if the lock could not be acquired.
*/
async releaseLock(lockKey: string): Promise<void> {}
async acquireLock(
lockKey: string,
ttlMs: number = 30000,
maxRetries: number = 3,
retryDelayMs: number = 100
): Promise<string | null> {
for (let attempt = 0; attempt < maxRetries; attempt++) {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
const token = `${instanceId}:${randomUUID()}`;
localLocks.set(lockKey, {
owner: token,
expiresAt: Date.now() + ttlMs
});
return token;
}
// The lock is currently held -- possibly by a different, unrelated
// caller in this same process. We intentionally do NOT treat
// same-process holders as automatically reentrant here: two
// independent logical operations (e.g. two different API requests)
// running concurrently in the same process must not both believe
// they hold the lock, or their writes under it can interleave
// unguarded. Just retry with backoff like any other contended lock.
if (attempt < maxRetries - 1) {
const delay = retryDelayMs * Math.pow(2, attempt);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
return null;
}
/**
* Release a lock previously acquired via acquireLock/acquireLockWithRetry.
* @param lockKey - Unique identifier for the lock
* @param token - the exact token returned by the acquisition being released.
* Required so a caller whose TTL already expired can't delete a
* different, currently-active holder's lock.
*/
async releaseLock(lockKey: string, token: string): Promise<void> {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (existing && existing.owner === token) {
localLocks.delete(lockKey);
}
}
/**
* Force release a lock regardless of owner (use with caution)
* @param lockKey - Unique identifier for the lock
*/
async forceReleaseLock(lockKey: string): Promise<void> {}
async forceReleaseLock(lockKey: string): Promise<void> {
localLocks.delete(lockKey);
}
/**
* Check if a lock exists and get its info
@@ -35,16 +94,44 @@ export class LockManager {
ttl: number;
owner?: string;
}> {
return { exists: true, ownedByMe: true, ttl: 0 };
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
return { exists: false, ownedByMe: false, ttl: 0 };
}
const ttl = Math.max(0, existing.expiresAt - Date.now());
return {
exists: true,
ownedByMe: existing.owner.startsWith(`${instanceId}:`),
ttl,
owner: existing.owner.split(":")[0]
};
}
/**
* Extend the TTL of an existing lock owned by this worker
* Extend the TTL of an existing lock, provided the token matches the
* acquisition currently holding it.
* @param lockKey - Unique identifier for the lock
* @param ttlMs - New TTL in milliseconds
* @param token - the token returned by the acquisition being extended
* @returns Promise<boolean> - true if extended successfully
*/
async extendLock(lockKey: string, ttlMs: number): Promise<boolean> {
async extendLock(
lockKey: string,
ttlMs: number,
token: string
): Promise<boolean> {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing || existing.owner !== token) {
return false;
}
existing.expiresAt = Date.now() + ttlMs;
localLocks.set(lockKey, existing);
return true;
}
@@ -54,15 +141,34 @@ export class LockManager {
* @param ttlMs - Time to live in milliseconds
* @param maxRetries - Maximum number of retry attempts
* @param baseDelayMs - Base delay between retries in milliseconds
* @returns Promise<boolean> - true if lock acquired
* @returns Promise<string | null> - token if acquired, null otherwise
*/
async acquireLockWithRetry(
lockKey: string,
ttlMs: number = 30000,
maxRetries: number = 5,
baseDelayMs: number = 100
): Promise<boolean> {
return true;
): Promise<string | null> {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
const acquired = await this.acquireLock(
lockKey,
ttlMs,
1,
baseDelayMs
);
if (acquired) {
return acquired;
}
if (attempt < maxRetries) {
const delay =
baseDelayMs * Math.pow(2, attempt) + Math.random() * 100;
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
return null;
}
/**
@@ -77,16 +183,16 @@ export class LockManager {
fn: () => Promise<T>,
ttlMs: number = 30000
): Promise<T> {
const acquired = await this.acquireLock(lockKey, ttlMs);
const token = await this.acquireLock(lockKey, ttlMs);
if (!acquired) {
if (!token) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
try {
return await fn();
} finally {
await this.releaseLock(lockKey);
await this.releaseLock(lockKey, token);
}
}
@@ -99,7 +205,21 @@ export class LockManager {
activeLocksCount: number;
locksOwnedByMe: number;
}> {
return { activeLocksCount: 0, locksOwnedByMe: 0 };
const now = Date.now();
for (const [key, value] of localLocks.entries()) {
if (value.expiresAt <= now) {
localLocks.delete(key);
}
}
let locksOwnedByMe = 0;
for (const value of localLocks.values()) {
if (value.owner.startsWith(`${instanceId}:`)) {
locksOwnedByMe++;
}
}
return { activeLocksCount: localLocks.size, locksOwnedByMe };
}
/**
@@ -0,0 +1,11 @@
import { z } from "zod";
export function createApiResponseSchema<T extends z.ZodTypeAny>(dataSchema: T) {
return z.object({
data: dataSchema.nullable(),
success: z.boolean(),
error: z.boolean(),
message: z.string(),
status: z.number()
});
}
+24
View File
@@ -0,0 +1,24 @@
export const ORG_REBUILD_CONCURRENCY_LIMIT = 10;
const orgActiveRebuilds = new Map<string, number>();
export async function incrementOrgRebuildCount(orgId: string): Promise<void> {
orgActiveRebuilds.set(orgId, (orgActiveRebuilds.get(orgId) ?? 0) + 1);
}
export async function decrementOrgRebuildCount(orgId: string): Promise<void> {
const current = orgActiveRebuilds.get(orgId) ?? 0;
if (current <= 1) {
orgActiveRebuilds.delete(orgId);
} else {
orgActiveRebuilds.set(orgId, current - 1);
}
}
export async function getOrgActiveRebuildCount(orgId: string): Promise<number> {
return orgActiveRebuilds.get(orgId) ?? 0;
}
export async function checkOrgRebuildRateLimit(orgId: string): Promise<boolean> {
return (orgActiveRebuilds.get(orgId) ?? 0) >= ORG_REBUILD_CONCURRENCY_LIMIT;
}
+74
View File
@@ -0,0 +1,74 @@
const MAX_RECURSION_DEPTH = 100;
const segmentRegexCache = new Map<string, RegExp>();
function getSegmentRegex(patternPart: string): RegExp {
let regex = segmentRegexCache.get(patternPart);
if (!regex) {
const regexPattern = patternPart
.replace(/[.+^${}()|[\]\\]/g, "\\$&")
.replace(/\*/g, ".*")
.replace(/\?/g, ".");
regex = new RegExp(`^${regexPattern}$`);
segmentRegexCache.set(patternPart, regex);
}
return regex;
}
export function isPathAllowed(pattern: string, path: string): boolean {
const normalize = (p: string) => p.split("/").filter(Boolean);
const patternParts = normalize(pattern);
const pathParts = normalize(path);
function matchSegments(
patternIndex: number,
pathIndex: number,
depth: number = 0
): boolean {
if (depth > MAX_RECURSION_DEPTH) {
return false;
}
const currentPatternPart = patternParts[patternIndex];
const currentPathPart = pathParts[pathIndex];
if (patternIndex >= patternParts.length) {
return pathIndex >= pathParts.length;
}
if (pathIndex >= pathParts.length) {
return patternParts.slice(patternIndex).every((p) => p === "*");
}
if (currentPatternPart === "*") {
if (matchSegments(patternIndex + 1, pathIndex, depth + 1)) {
return true;
}
if (matchSegments(patternIndex, pathIndex + 1, depth + 1)) {
return true;
}
return false;
}
if (currentPatternPart.includes("*")) {
const regex = getSegmentRegex(currentPatternPart);
if (regex.test(currentPathPart)) {
return matchSegments(
patternIndex + 1,
pathIndex + 1,
depth + 1
);
}
return false;
}
if (currentPatternPart !== currentPathPart) {
return false;
}
return matchSegments(patternIndex + 1, pathIndex + 1, depth + 1);
}
return matchSegments(0, 0, 0);
}
File diff suppressed because it is too large Load Diff
+148
View File
@@ -0,0 +1,148 @@
import logger from "@server/logger";
import { isTransientError } from "@server/lib/dbRetry";
export type RebuildJobType = "site-resource" | "client";
export interface RebuildJob {
type: RebuildJobType;
id: number;
// Number of times this job has already been re-queued after a transient
// failure. Absent/0 means it has not failed yet.
attempt?: number;
}
export interface RebuildJobHandlers {
onSiteResource(siteResourceId: number): Promise<void>;
onClient(clientId: number): Promise<void>;
}
export interface RebuildQueueManager {
enqueue(job: RebuildJob): Promise<void>;
startProcessing(handlers: RebuildJobHandlers): void;
isQueued(job: RebuildJob): Promise<boolean>;
}
// In-process FIFO used when there is no Redis to back a distributed queue
// (OSS build, or Redis unavailable). A job that loses the per-resource
// rebuild lock race lands here instead of being silently dropped, and gets
// retried shortly after against fresh DB state.
const POLL_INTERVAL_MS = 500;
const BATCH_SIZE = 5;
// A job that fails with a transient DB error gets re-queued with backoff
// instead of being dropped, up to this many times.
const MAX_JOB_ATTEMPTS = 5;
const JOB_RETRY_BASE_DELAY_MS = 1000;
function dedupeKey(job: RebuildJob): string {
return `${job.type}:${job.id}`;
}
class InMemoryRebuildQueue implements RebuildQueueManager {
private queue: RebuildJob[] = [];
private queuedSet = new Set<string>();
private processing = false;
private processingStarted = false;
private handlers: RebuildJobHandlers | null = null;
async isQueued(job: RebuildJob): Promise<boolean> {
return this.queuedSet.has(dedupeKey(job));
}
async enqueue(job: RebuildJob): Promise<void> {
const key = dedupeKey(job);
if (this.queuedSet.has(key)) {
logger.debug(
`Rebuild queue: skipped duplicate queued job ${job.type}:${job.id}`
);
return;
}
this.queuedSet.add(key);
this.queue.push(job);
logger.debug(
`Rebuild queue: enqueued ${job.type}:${job.id} (queue position: tail)`
);
}
startProcessing(handlers: RebuildJobHandlers): void {
if (this.processingStarted) return;
this.processingStarted = true;
this.handlers = handlers;
setInterval(() => {
this.tryProcessBatch().catch((err) => {
logger.error(
"Rebuild queue: unhandled error in process loop:",
err
);
});
}, POLL_INTERVAL_MS);
logger.info("Rebuild queue processor started (in-memory)");
}
private async tryProcessBatch(): Promise<void> {
if (this.processing || !this.handlers || this.queue.length === 0) {
return;
}
this.processing = true;
try {
for (let i = 0; i < BATCH_SIZE; i++) {
const job = this.queue.shift();
if (!job) break; // queue drained
// Remove from the dedupe set once dequeued so the same job
// can be re-queued while this one is in progress.
this.queuedSet.delete(dedupeKey(job));
logger.debug(
`Rebuild queue: processing ${job.type}:${job.id}`
);
try {
if (job.type === "site-resource") {
await this.handlers.onSiteResource(job.id);
} else if (job.type === "client") {
await this.handlers.onClient(job.id);
} else {
logger.warn(
`Rebuild queue: unknown job type "${(job as any).type}", discarding`
);
}
logger.debug(
`Rebuild queue: completed ${job.type}:${job.id}`
);
} catch (err) {
const attempt = (job.attempt ?? 0) + 1;
if (isTransientError(err) && attempt <= MAX_JOB_ATTEMPTS) {
const delay =
JOB_RETRY_BASE_DELAY_MS * Math.pow(2, attempt - 1);
logger.warn(
`Rebuild queue: job ${job.type}:${job.id} hit a transient error (attempt ${attempt}/${MAX_JOB_ATTEMPTS}), re-queuing in ${delay}ms:`,
err
);
setTimeout(() => {
this.enqueue({ ...job, attempt }).catch(
(enqueueErr) =>
logger.error(
`Rebuild queue: failed to re-queue ${job.type}:${job.id} after transient error:`,
enqueueErr
)
);
}, delay);
} else {
logger.error(
`Rebuild queue: job ${job.type}:${job.id} threw an error:`,
err
);
}
}
}
} finally {
this.processing = false;
}
}
}
export const rebuildQueue: RebuildQueueManager = new InMemoryRebuildQueue();
+11
View File
@@ -0,0 +1,11 @@
export function getFirstString(value: unknown): string | undefined {
if (typeof value === "string") {
return value;
}
if (Array.isArray(value) && typeof value[0] === "string") {
return value[0];
}
return undefined;
}
+30 -4
View File
@@ -1,6 +1,6 @@
import { z } from "zod";
import { db, logsDb, statusHistory } from "@server/db";
import { and, eq, gte, asc } from "drizzle-orm";
import { and, eq, gte, lt, asc, desc } from "drizzle-orm";
import { regionalCache as cache } from "#dynamic/lib/cache";
const STATUS_HISTORY_CACHE_TTL = 60; // seconds
@@ -42,7 +42,29 @@ export async function getCachedStatusHistory(
)
.orderBy(asc(statusHistory.timestamp));
const { buckets, totalDowntime } = computeBuckets(events, days);
// Fetch the last known state before the window so that entities that
// haven't changed status recently still show the correct status rather
// than appearing as "no_data".
const [lastKnownEvent] = await logsDb
.select()
.from(statusHistory)
.where(
and(
eq(statusHistory.entityType, entityType),
eq(statusHistory.entityId, entityId),
lt(statusHistory.timestamp, startSec)
)
)
.orderBy(desc(statusHistory.timestamp))
.limit(1);
const priorStatus = lastKnownEvent?.status ?? null;
const { buckets, totalDowntime } = computeBuckets(
events,
days,
priorStatus
);
const totalWindow = days * 86400;
const overallUptime =
totalWindow > 0
@@ -110,7 +132,8 @@ export function computeBuckets(
timestamp: number;
id: number;
}[],
days: number
days: number,
priorStatus: string | null = null
): { buckets: StatusHistoryDayBucket[]; totalDowntime: number } {
const nowSec = Math.floor(Date.now() / 1000);
@@ -136,7 +159,10 @@ export function computeBuckets(
.filter((e) => e.timestamp < dayStartSec)
.at(-1);
const currentStatus = lastBeforeDay?.status ?? null;
// Fall back to the last known state before the entire query window
// so that entities that haven't generated events recently still show
// as their actual status rather than "no_data".
const currentStatus = lastBeforeDay?.status ?? priorStatus ?? null;
const windows: { start: number; end: number | null; status: string }[] =
[];
+14 -4
View File
@@ -2,7 +2,14 @@ import { PostHog } from "posthog-node";
import config from "./config";
import { getHostMeta } from "./hostMeta";
import logger from "@server/logger";
import { alertRules, apiKeys, blueprints, db, roles, siteResources } from "@server/db";
import {
alertRules,
apiKeys,
blueprints,
db,
roles,
siteResources
} from "@server/db";
import { sites, users, orgs, resources, clients, idp } from "@server/db";
import { eq, count, notInArray, and, isNotNull, isNull } from "drizzle-orm";
import { APP_VERSION } from "./consts";
@@ -143,8 +150,7 @@ class TelemetryClient {
.select({
name: resources.name,
sso: resources.sso,
protocol: resources.protocol,
http: resources.http
mode: resources.mode
})
.from(resources);
@@ -175,6 +181,7 @@ class TelemetryClient {
let numPrivResourceHosts = 0;
let numPrivResourceCidr = 0;
let numPrivResourceHttp = 0;
let numPrivResourceSsh = 0;
for (const res of allPrivateResources) {
if (res.mode === "host") {
numPrivResourceHosts += 1;
@@ -182,6 +189,8 @@ class TelemetryClient {
numPrivResourceCidr += 1;
} else if (res.mode === "http") {
numPrivResourceHttp += 1;
} else if (res.mode === "ssh") {
numPrivResourceSsh += 1;
}
if (res.alias) {
@@ -201,6 +210,7 @@ class TelemetryClient {
numPrivateResourceHosts: numPrivResourceHosts,
numPrivateResourceCidr: numPrivResourceCidr,
numPrivateResourceHttp: numPrivResourceHttp,
numPrivateResourceSsh: numPrivResourceSsh,
numAlertRules: numAlertRules.count,
numUserDevices: userDevicesCount.count,
numMachineClients: machineClients.count,
@@ -311,7 +321,7 @@ class TelemetryClient {
(r) => r.sso
).length,
num_resources_non_http: stats.resources.filter(
(r) => !r.http
(r) => r.mode !== "http"
).length,
num_newt_sites: stats.sites.filter((s) => s.type === "newt")
.length,
+9 -1
View File
@@ -511,6 +511,12 @@ export class TraefikConfigManager {
let traefikConfig;
try {
const currentExitNode = await getCurrentExitNodeId();
const maintenancePort = config.getRawConfig().server.next_port;
const maintenanceHost =
config.getRawConfig().server.internal_hostname;
const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`;
// logger.debug(`Fetching traefik config for exit node: ${currentExitNode}`);
traefikConfig = await getTraefikConfig(
// this is called by the local exit node to get its own config
@@ -520,7 +526,9 @@ export class TraefikConfigManager {
build != "oss", // generate the login pages on the cloud and hybrid,
build == "saas"
? false
: config.getRawConfig().traefik.allow_raw_resources // dont allow raw resources on saas otherwise use config
: config.getRawConfig().traefik.allow_raw_resources, // dont allow raw resources on saas otherwise use config
pangolinUIUrl, // generate maintenance pages on cloud and hybrid
pangolinUIUrl // generate browser gateway targets on cloud and hybrid
);
const domains = new Set<string>();
+9 -10
View File
@@ -44,7 +44,8 @@ export async function getTraefikConfig(
filterOutNamespaceDomains = false, // UNUSED BUT USED IN PRIVATE
generateLoginPageRouters = false, // UNUSED BUT USED IN PRIVATE
allowRawResources = true,
allowMaintenancePage = true // UNUSED BUT USED IN PRIVATE
maintenancePageUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE
browserGatewayUiUrl: string | null = null // UNUSED BUT USED IN PRIVATE
): Promise<any> {
// Get resources with their targets and sites in a single optimized query
// Start from sites on this exit node, then join to targets and resources
@@ -55,9 +56,7 @@ export async function getTraefikConfig(
resourceName: resources.name,
fullDomain: resources.fullDomain,
ssl: resources.ssl,
http: resources.http,
proxyPort: resources.proxyPort,
protocol: resources.protocol,
subdomain: resources.subdomain,
domainId: resources.domainId,
enabled: resources.enabled,
@@ -69,6 +68,7 @@ export async function getTraefikConfig(
responseHeaders: resources.responseHeaders,
proxyProtocol: resources.proxyProtocol,
proxyProtocolVersion: resources.proxyProtocolVersion,
mode: resources.mode,
// Target fields
targetId: targets.targetId,
@@ -116,8 +116,8 @@ export async function getTraefikConfig(
),
inArray(sites.type, siteTypes),
allowRawResources
? isNotNull(resources.http) // ignore the http check if allow_raw_resources is true
: eq(resources.http, true)
? inArray(resources.mode, ["http", "udp", "tcp"]) // allow all three
: eq(resources.mode, "http")
)
)
.orderBy(desc(targets.priority), targets.targetId); // stable ordering
@@ -167,9 +167,8 @@ export async function getTraefikConfig(
key: key,
fullDomain: row.fullDomain,
ssl: row.ssl,
http: row.http,
mode: row.mode,
proxyPort: row.proxyPort,
protocol: row.protocol,
subdomain: row.subdomain,
domainId: row.domainId,
enabled: row.enabled,
@@ -244,7 +243,7 @@ export async function getTraefikConfig(
continue;
}
if (resource.http) {
if (resource.mode === "http") {
if (!resource.domainId || !resource.fullDomain) {
continue;
}
@@ -597,13 +596,13 @@ export async function getTraefikConfig(
serviceName
].loadBalancer.serversTransport = transportName;
}
} else {
} else if (resource.mode === "tcp" || resource.mode === "udp") {
// Non-HTTP (TCP/UDP) configuration
if (!resource.enableProxy || !resource.proxyPort) {
continue;
}
const protocol = resource.protocol.toLowerCase();
const protocol = resource.mode === "udp" ? "udp" : "tcp"; // all of the other ones are tcp
const port = resource.proxyPort;
if (!port) {
+3 -3
View File
@@ -14,7 +14,7 @@ import {
} from "@server/db";
import { eq, and, inArray, ne, exists } from "drizzle-orm";
import { usageService } from "@server/lib/billing/usageService";
import { FeatureId } from "@server/lib/billing";
import { LimitId } from "@server/lib/billing";
export async function assignUserToOrg(
org: Org,
@@ -61,7 +61,7 @@ export async function assignUserToOrg(
);
if (orgsInBillingDomainThatTheUserIsStillIn.length === 0) {
await usageService.add(org.orgId, FeatureId.USERS, 1, trx);
await usageService.add(org.orgId, LimitId.USERS, 1, trx);
}
}
}
@@ -157,7 +157,7 @@ export async function removeUserFromOrg(
);
if (orgsInBillingDomainThatTheUserIsStillIn.length === 0) {
await usageService.add(org.orgId, FeatureId.USERS, -1, trx);
await usageService.add(org.orgId, LimitId.USERS, -1, trx);
}
}
}
+42 -1
View File
@@ -1,4 +1,7 @@
import { isValidUrlGlobPattern } from "./validators";
import {
getResourceRuleValueValidationError,
isValidUrlGlobPattern
} from "./validators";
import { assertEquals } from "@test/assert";
function runTests() {
@@ -236,6 +239,43 @@ function runTests() {
"Path with isolated percent sign should be invalid"
);
// ASN validation tests
assertEquals(
getResourceRuleValueValidationError("ASN", "AS15169"),
null,
"Standard ASN should be valid"
);
assertEquals(
getResourceRuleValueValidationError("ASN", " As15169 "),
null,
"Standard ASN should be valid with mixed case and whitespace"
);
assertEquals(
getResourceRuleValueValidationError("ASN", "ALL"),
null,
"ALL ASN selector should be valid"
);
assertEquals(
getResourceRuleValueValidationError("ASN", " all "),
null,
"ALL ASN selector should be valid with mixed case and whitespace"
);
assertEquals(
getResourceRuleValueValidationError("ASN", "AS0"),
null,
"AS0 alias should be valid"
);
assertEquals(
getResourceRuleValueValidationError("ASN", " as0 "),
null,
"AS0 alias should be valid with mixed case and whitespace"
);
assertEquals(
getResourceRuleValueValidationError("ASN", "not-an-asn"),
"Invalid ASN provided",
"Invalid ASN should return an error"
);
console.log("All tests passed!");
}
@@ -244,4 +284,5 @@ try {
runTests();
} catch (error) {
console.error("Test failed:", error);
process.exit(1);
}
+44
View File
@@ -1,5 +1,7 @@
import z from "zod";
import ipaddr from "ipaddr.js";
import { COUNTRIES } from "@server/db/countries";
import { isValidRegionId } from "@server/db/regions";
export function isValidCIDR(cidr: string): boolean {
return (
@@ -67,6 +69,48 @@ export function isValidUrlGlobPattern(pattern: string): boolean {
return true;
}
export const RESOURCE_RULE_MATCH_TYPES = [
"CIDR",
"IP",
"PATH",
"COUNTRY",
"ASN",
"REGION"
] as const;
export type ResourceRuleMatchType = (typeof RESOURCE_RULE_MATCH_TYPES)[number];
export function getResourceRuleValueValidationError(
match: ResourceRuleMatchType,
value: string
): string | null {
switch (match) {
case "CIDR":
return isValidCIDR(value) ? null : "Invalid CIDR provided";
case "IP":
return isValidIP(value) ? null : "Invalid IP provided";
case "PATH":
return isValidUrlGlobPattern(value)
? null
: "Invalid URL glob pattern provided";
case "REGION":
return isValidRegionId(value) ? null : "Invalid region ID provided";
case "COUNTRY":
return COUNTRIES.some((country) => country.code === value)
? null
: "Invalid country code provided";
case "ASN":
const normalizedValue = value.trim().toUpperCase();
return /^AS\d+$/.test(normalizedValue) ||
normalizedValue === "ALL" ||
normalizedValue === "AS0"
? null
: "Invalid ASN provided";
default:
return "Invalid rule match type provided";
}
}
export function isUrlValid(url: string | undefined) {
if (!url) return true; // the link is optional in the schema so if it's empty it's valid
var pattern = new RegExp(
+2
View File
@@ -28,7 +28,9 @@ export * from "./verifyApiKeyAccess";
export * from "./verifySiteProvisioningKeyAccess";
export * from "./verifyDomainAccess";
export * from "./verifyUserIsOrgOwner";
export * from "./verifyUserFromResourceSession";
export * from "./verifySiteResourceAccess";
export * from "./logActionAudit";
export * from "./verifyOlmAccess";
export * from "./verifyLimits";
export * from "./verifyResourcePolicyAccess";
+1
View File
@@ -16,3 +16,4 @@ export * from "./verifyApiKeyClientAccess";
export * from "./verifyApiKeySiteResourceAccess";
export * from "./verifyApiKeyIdpAccess";
export * from "./verifyApiKeyDomainAccess";
export * from "./verifyApiKeyResourcePolicyAccess";
@@ -4,6 +4,7 @@ import { resourceAccessToken, resources, apiKeyOrg } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyAccessTokenAccess(
req: Request,
@@ -12,7 +13,7 @@ export async function verifyApiKeyAccessTokenAccess(
) {
try {
const apiKey = req.apiKey;
const accessTokenId = req.params.accessTokenId;
const accessTokenId = getFirstString(req.params.accessTokenId);
if (!apiKey) {
return next(
@@ -20,6 +21,12 @@ export async function verifyApiKeyAccessTokenAccess(
);
}
if (!accessTokenId) {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Invalid access token ID")
);
}
const [accessToken] = await db
.select()
.from(resourceAccessToken)
@@ -4,6 +4,7 @@ import { apiKeys, apiKeyOrg } from "@server/db";
import { and, eq, or } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyApiKeyAccess(
req: Request,
@@ -14,8 +15,10 @@ export async function verifyApiKeyApiKeyAccess(
const { apiKey: callerApiKey } = req;
const apiKeyId =
req.params.apiKeyId || req.body.apiKeyId || req.query.apiKeyId;
const orgId = req.params.orgId;
getFirstString(req.params.apiKeyId) ||
getFirstString(req.body.apiKeyId) ||
getFirstString(req.query.apiKeyId);
const orgId = getFirstString(req.params.orgId);
if (!callerApiKey) {
return next(
@@ -3,6 +3,7 @@ import { db, domains, orgDomains, apiKeyOrg } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyDomainAccess(
req: Request,
@@ -12,8 +13,10 @@ export async function verifyApiKeyDomainAccess(
try {
const apiKey = req.apiKey;
const domainId =
req.params.domainId || req.body.domainId || req.query.domainId;
const orgId = req.params.orgId;
getFirstString(req.params.domainId) ||
getFirstString(req.body.domainId) ||
getFirstString(req.query.domainId);
const orgId = getFirstString(req.params.orgId);
if (!apiKey) {
return next(
@@ -27,6 +30,12 @@ export async function verifyApiKeyDomainAccess(
);
}
if (!orgId) {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Invalid organization ID")
);
}
if (apiKey.isRoot) {
// Root keys can access any domain in any org
return next();
@@ -4,6 +4,7 @@ import { idp, idpOrg, apiKeyOrg } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyIdpAccess(
req: Request,
@@ -12,8 +13,12 @@ export async function verifyApiKeyIdpAccess(
) {
try {
const apiKey = req.apiKey;
const idpId = req.params.idpId || req.body.idpId || req.query.idpId;
const orgId = req.params.orgId;
const idpIdRaw =
getFirstString(req.params.idpId) ||
getFirstString(req.body.idpId) ||
getFirstString(req.query.idpId);
const idpId = Number.parseInt(idpIdRaw ?? "", 10);
const orgId = getFirstString(req.params.orgId);
if (!apiKey) {
return next(
@@ -27,7 +32,7 @@ export async function verifyApiKeyIdpAccess(
);
}
if (!idpId) {
if (Number.isNaN(idpId)) {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Invalid IDP ID")
);
@@ -4,6 +4,7 @@ import { apiKeyOrg } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyOrgAccess(
req: Request,
@@ -12,7 +13,7 @@ export async function verifyApiKeyOrgAccess(
) {
try {
const apiKeyId = req.apiKey?.apiKeyId;
const orgId = req.params.orgId;
const orgId = getFirstString(req.params.orgId);
if (!apiKeyId) {
return next(
@@ -45,7 +46,7 @@ export async function verifyApiKeyOrgAccess(
}
if (!req.apiKeyOrg) {
next(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Key does not have access to this organization"
@@ -0,0 +1,92 @@
import { Request, Response, NextFunction } from "express";
import { db } from "@server/db";
import { resourcePolicies, apiKeyOrg } from "@server/db";
import { eq, and } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
export async function verifyApiKeyResourcePolicyAccess(
req: Request,
res: Response,
next: NextFunction
) {
const apiKey = req.apiKey;
const resourcePolicyId =
req.params.resourcePolicyId ||
req.body.resourcePolicyId ||
req.query.resourcePolicyId;
if (!apiKey) {
return next(
createHttpError(HttpCode.UNAUTHORIZED, "Key not authenticated")
);
}
try {
// Retrieve the resource policy
const [policy] = await db
.select()
.from(resourcePolicies)
.where(eq(resourcePolicies.resourcePolicyId, resourcePolicyId))
.limit(1);
if (!policy) {
return next(
createHttpError(
HttpCode.NOT_FOUND,
`Resource policy with ID ${resourcePolicyId} not found`
)
);
}
if (apiKey.isRoot) {
// Root keys can access any resource policy in any org
return next();
}
if (!policy.orgId) {
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
`Resource policy with ID ${resourcePolicyId} does not have an organization ID`
)
);
}
// Verify that the API key is linked to the resource policy's organization
if (!req.apiKeyOrg) {
const apiKeyOrgResult = await db
.select()
.from(apiKeyOrg)
.where(
and(
eq(apiKeyOrg.apiKeyId, apiKey.apiKeyId),
eq(apiKeyOrg.orgId, policy.orgId)
)
)
.limit(1);
if (apiKeyOrgResult.length > 0) {
req.apiKeyOrg = apiKeyOrgResult[0];
}
}
if (!req.apiKeyOrg) {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Key does not have access to this organization"
)
);
}
return next();
} catch (error) {
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"Error verifying resource policy access"
)
);
}
}
@@ -4,6 +4,7 @@ import { siteResources, apiKeyOrg } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeySiteResourceAccess(
req: Request,
@@ -12,7 +13,8 @@ export async function verifyApiKeySiteResourceAccess(
) {
try {
const apiKey = req.apiKey;
const siteResourceId = parseInt(req.params.siteResourceId);
const siteResourceIdRaw = getFirstString(req.params.siteResourceId);
const siteResourceId = Number.parseInt(siteResourceIdRaw ?? "", 10);
if (!apiKey) {
return next(
@@ -20,7 +22,7 @@ export async function verifyApiKeySiteResourceAccess(
);
}
if (!siteResourceId) {
if (Number.isNaN(siteResourceId)) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
@@ -4,6 +4,7 @@ import { resources, targets, apiKeyOrg } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyTargetAccess(
req: Request,
@@ -12,7 +13,8 @@ export async function verifyApiKeyTargetAccess(
) {
try {
const apiKey = req.apiKey;
const targetId = parseInt(req.params.targetId);
const targetIdRaw = getFirstString(req.params.targetId);
const targetId = Number.parseInt(targetIdRaw ?? "", 10);
if (!apiKey) {
return next(
@@ -20,7 +22,7 @@ export async function verifyApiKeyTargetAccess(
);
}
if (isNaN(targetId)) {
if (Number.isNaN(targetId)) {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Invalid target ID")
);
+10 -4
View File
@@ -7,6 +7,7 @@ import HttpCode from "@server/types/HttpCode";
import { canUserAccessResource } from "@server/auth/canUserAccessResource";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyAccessTokenAccess(
req: Request,
@@ -14,7 +15,7 @@ export async function verifyAccessTokenAccess(
next: NextFunction
) {
const userId = req.user!.userId;
const accessTokenId = req.params.accessTokenId;
const accessTokenId = getFirstString(req.params.accessTokenId);
if (!userId) {
return next(
@@ -22,6 +23,12 @@ export async function verifyAccessTokenAccess(
);
}
if (!accessTokenId) {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Invalid access token ID")
);
}
const [accessToken] = await db
.select()
.from(resourceAccessToken)
@@ -87,7 +94,7 @@ export async function verifyAccessTokenAccess(
}
if (!req.userOrg) {
next(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"User does not have access to this organization"
@@ -112,8 +119,7 @@ export async function verifyAccessTokenAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+1 -2
View File
@@ -56,8 +56,7 @@ export async function verifyAdmin(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+21 -9
View File
@@ -6,6 +6,7 @@ import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyApiKeyAccess(
req: Request,
@@ -14,9 +15,24 @@ export async function verifyApiKeyAccess(
) {
try {
const userId = req.user!.userId;
const apiKeyId =
req.params.apiKeyId || req.body.apiKeyId || req.query.apiKeyId;
const orgId = req.params.orgId;
const apiKeyIdFromParams = getFirstString(req.params?.apiKeyId);
const apiKeyIdFromBody = getFirstString(req.body?.apiKeyId);
if (
apiKeyIdFromParams &&
apiKeyIdFromBody &&
apiKeyIdFromParams !== apiKeyIdFromBody
) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
"API key ID provided in both URL and body with different values"
)
);
}
const apiKeyId = apiKeyIdFromParams || apiKeyIdFromBody;
const orgId = getFirstString(req.params.orgId);
if (!userId) {
return next(
@@ -97,17 +113,13 @@ export async function verifyApiKeyAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
}
req.userOrgRoleIds = await getUserOrgRoleIds(
req.userOrg.userId,
orgId
);
req.userOrgRoleIds = await getUserOrgRoleIds(req.userOrg.userId, orgId);
return next();
} catch (error) {
+2 -6
View File
@@ -107,8 +107,7 @@ export async function verifyClientAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
@@ -129,10 +128,7 @@ export async function verifyClientAccess(
.where(
and(
eq(roleClients.clientId, client.clientId),
inArray(
roleClients.roleId,
req.userOrgRoleIds!
)
inArray(roleClients.roleId, req.userOrgRoleIds!)
)
)
.limit(1)
+5 -9
View File
@@ -6,6 +6,7 @@ import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyDomainAccess(
req: Request,
@@ -14,9 +15,8 @@ export async function verifyDomainAccess(
) {
try {
const userId = req.user!.userId;
const domainId =
req.params.domainId;
const orgId = req.params.orgId;
const domainId = getFirstString(req.params.domainId);
const orgId = getFirstString(req.params.orgId);
if (!userId) {
return next(
@@ -62,10 +62,7 @@ export async function verifyDomainAccess(
.select()
.from(userOrgs)
.where(
and(
eq(userOrgs.userId, userId),
eq(userOrgs.orgId, orgId)
)
and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId))
)
.limit(1);
req.userOrg = userOrgRole[0];
@@ -91,8 +88,7 @@ export async function verifyDomainAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+5 -1
View File
@@ -3,6 +3,7 @@ import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { usageService } from "@server/lib/billing/usageService";
import { build } from "@server/build";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyLimits(
req: Request,
@@ -13,7 +14,10 @@ export async function verifyLimits(
return next();
}
const orgId = req.userOrgId || req.apiKeyOrg?.orgId || req.params.orgId;
const orgId =
req.userOrgId ||
req.apiKeyOrg?.orgId ||
getFirstString(req.params.orgId);
if (!orgId) {
return next(); // its fine if we silently fail here because this is not critical to operation or security and its better user experience if we dont fail
+4 -3
View File
@@ -6,6 +6,8 @@ import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getFirstString } from "@server/lib/requestParams";
import logger from "@server/logger";
export async function verifyOrgAccess(
req: Request,
@@ -13,7 +15,7 @@ export async function verifyOrgAccess(
next: NextFunction
) {
const userId = req.user!.userId;
const orgId = req.params.orgId;
const orgId = getFirstString(req.params.orgId);
if (!userId) {
return next(
@@ -58,8 +60,7 @@ export async function verifyOrgAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+18 -30
View File
@@ -1,11 +1,15 @@
import { Request, Response, NextFunction } from "express";
import { db, Resource } from "@server/db";
import { resources, userOrgs, userResources, roleResources } from "@server/db";
import { and, eq, inArray } from "drizzle-orm";
import { resources, userOrgs } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import {
getRoleResourceAccess,
getUserResourceAccess
} from "@server/db/queries/verifySessionQueries";
export async function verifyResourceAccess(
req: Request,
@@ -101,8 +105,7 @@ export async function verifyResourceAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
@@ -116,37 +119,22 @@ export async function verifyResourceAccess(
const roleResourceAccess =
(req.userOrgRoleIds?.length ?? 0) > 0
? await db
.select()
.from(roleResources)
.where(
and(
eq(roleResources.resourceId, resource.resourceId),
inArray(
roleResources.roleId,
req.userOrgRoleIds!
)
)
)
.limit(1)
: [];
? await getRoleResourceAccess(
resource.resourceId,
req.userOrgRoleIds!
)
: null;
if (roleResourceAccess.length > 0) {
if (roleResourceAccess) {
return next();
}
const userResourceAccess = await db
.select()
.from(userResources)
.where(
and(
eq(userResources.userId, userId),
eq(userResources.resourceId, resource.resourceId)
)
)
.limit(1);
const userResourceAccess = await getUserResourceAccess(
userId,
resource.resourceId
);
if (userResourceAccess.length > 0) {
if (userResourceAccess) {
return next();
}
@@ -0,0 +1,126 @@
import { Request, Response, NextFunction } from "express";
import { db } from "@server/db";
import { resourcePolicies, userOrgs } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
export async function verifyResourcePolicyAccess(
req: Request,
res: Response,
next: NextFunction
) {
const userId = req.user!.userId;
const resourcePolicyIdStr =
req.params?.resourcePolicyId ||
req.body?.resourcePolicyId ||
req.query?.resourcePolicyId;
const niceId = req.params?.niceId || req.body?.niceId || req.query?.niceId;
const orgId = req.params?.orgId || req.body?.orgId || req.query?.orgId;
try {
if (!userId) {
return next(
createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated")
);
}
let policy: typeof resourcePolicies.$inferSelect | null = null;
if (orgId && niceId) {
const [policyRes] = await db
.select()
.from(resourcePolicies)
.where(
and(
eq(resourcePolicies.niceId, niceId),
eq(resourcePolicies.orgId, orgId)
)
)
.limit(1);
policy = policyRes ?? null;
} else {
const resourcePolicyId = parseInt(resourcePolicyIdStr);
if (isNaN(resourcePolicyId)) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
"Invalid resource policy ID"
)
);
}
const [policyRes] = await db
.select()
.from(resourcePolicies)
.where(eq(resourcePolicies.resourcePolicyId, resourcePolicyId))
.limit(1);
policy = policyRes ?? null;
}
if (!policy) {
return next(
createHttpError(
HttpCode.NOT_FOUND,
`Resource policy with ID ${resourcePolicyIdStr ?? niceId} not found`
)
);
}
if (!req.userOrg) {
const userOrgRes = await db
.select()
.from(userOrgs)
.where(
and(
eq(userOrgs.userId, userId),
eq(userOrgs.orgId, policy.orgId)
)
)
.limit(1);
req.userOrg = userOrgRes[0];
}
if (!req.userOrg || req.userOrg.orgId !== policy.orgId) {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"User does not have access to this organization"
)
);
}
if (req.orgPolicyAllowed === undefined && req.userOrg.orgId) {
const policyCheck = await checkOrgAccessPolicy({
orgId: req.userOrg.orgId,
userId,
session: req.session
});
req.orgPolicyAllowed = policyCheck.allowed;
if (!policyCheck.allowed || policyCheck.error) {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"" + (policyCheck.error || "Unknown error")
)
);
}
}
req.userOrgRoleIds = await getUserOrgRoleIds(
req.userOrg.userId,
policy.orgId
);
req.userOrgId = policy.orgId;
return next();
} catch (error) {
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"Error verifying resource policy access"
)
);
}
}
+1 -2
View File
@@ -132,8 +132,7 @@ export async function verifyRoleAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
@@ -45,8 +45,7 @@ export async function verifySetResourceClients(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+1 -2
View File
@@ -40,8 +40,7 @@ export async function verifySetResourceUsers(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+322
View File
@@ -0,0 +1,322 @@
import { assertEquals } from "@test/assert";
/**
* Tests for the cross-organization site binding prevention in verifySiteAccess.
*
* verifySiteAccess now includes a check: if req.userOrgId is already set by a
* previous middleware (e.g. verifyResourceAccess or verifyTargetAccess), and the
* loaded site's orgId differs from req.userOrgId, the request is rejected with
* 403 Forbidden.
*
* Route stacks after fix:
* PUT /resource/:resourceId/target
* → verifyResourceAccess → verifySiteAccess → verifyLimits → ...
* POST /target/:targetId
* → verifyTargetAccess → verifySiteAccess → verifyLimits → ...
*
* verifyResourceAccess sets req.userOrgId to the resource's org.
* verifyTargetAccess sets req.userOrgId to the target's resource org.
* verifySiteAccess then checks site.orgId against req.userOrgId before
* overwriting it with the site's org.
*/
// --- Core org-matching logic (mirrors the check in verifySiteAccess) ---
function siteOrgMatchesExpectedOrg(
siteOrgId: string | null | undefined,
expectedOrgId: string | null | undefined
): boolean {
if (!siteOrgId || !expectedOrgId) {
return false;
}
return siteOrgId === expectedOrgId;
}
// Simulates the condition check in verifySiteAccess:
// if (req.userOrgId && site.orgId !== req.userOrgId) { reject }
function shouldRejectCrossOrgSite(
siteOrgId: string,
reqUserOrgId: string | undefined
): boolean {
// The actual check in verifySiteAccess is:
// if (req.userOrgId && site.orgId !== req.userOrgId) { reject }
return !!(reqUserOrgId && siteOrgId !== reqUserOrgId);
}
// --- Tests ---
function testSiteOrgMatchLogic() {
console.log("Running verifySiteAccess org-match logic tests...");
// Test 1: Same org — should match
{
const result = siteOrgMatchesExpectedOrg(
"org-attacker",
"org-attacker"
);
assertEquals(result, true, "Same org should match");
}
// Test 2: Different org — should NOT match (cross-org bypass scenario)
{
const result = siteOrgMatchesExpectedOrg("org-victim", "org-attacker");
assertEquals(
result,
false,
"Cross-org site should NOT match expected org"
);
}
// Test 3: Site orgId is null — should NOT match
{
const result = siteOrgMatchesExpectedOrg(null, "org-attacker");
assertEquals(result, false, "Null site orgId should NOT match");
}
// Test 4: Expected orgId is null — should NOT match
{
const result = siteOrgMatchesExpectedOrg("org-attacker", null);
assertEquals(result, false, "Null expected orgId should NOT match");
}
// Test 5: Both null — should NOT match
{
const result = siteOrgMatchesExpectedOrg(null, null);
assertEquals(result, false, "Both null should NOT match");
}
// Test 6: Empty string orgIds — should NOT match (empty string is falsy)
{
const result = siteOrgMatchesExpectedOrg("", "org-attacker");
assertEquals(result, false, "Empty site orgId should NOT match");
}
// Test 7: Undefined orgIds — should NOT match
{
const result = siteOrgMatchesExpectedOrg(undefined, "org-attacker");
assertEquals(result, false, "Undefined site orgId should NOT match");
}
console.log("All verifySiteAccess org-match logic tests passed.");
}
function testShouldRejectCrossOrgSite() {
console.log(
"Running shouldRejectCrossOrgSite tests (mirrors verifySiteAccess check)..."
);
// Test: No prior org context (undefined) — should NOT reject
// This is the normal case for site-only routes (e.g. PUT /site/:siteId)
// where verifySiteAccess runs without a prior verifyResourceAccess.
{
const shouldReject = shouldRejectCrossOrgSite("org-victim", undefined);
assertEquals(
shouldReject,
false,
"No prior org context should NOT reject (normal site routes)"
);
}
// Test: Same org — should NOT reject
{
const shouldReject = shouldRejectCrossOrgSite(
"org-attacker",
"org-attacker"
);
assertEquals(shouldReject, false, "Same org should NOT reject");
}
// Test: Different org — should reject
{
const shouldReject = shouldRejectCrossOrgSite(
"org-victim",
"org-attacker"
);
assertEquals(shouldReject, true, "Cross-org site should be rejected");
}
// Test: Empty string userOrgId — should NOT reject (falsy, check is skipped)
{
const shouldReject = shouldRejectCrossOrgSite("org-victim", "");
assertEquals(
shouldReject,
false,
"Empty string userOrgId should NOT reject (check is skipped)"
);
}
console.log("All shouldRejectCrossOrgSite tests passed.");
}
// --- Route stack validation tests ---
function testRouteStackOrdering() {
console.log("Running route stack ordering tests...");
const createTargetStack = [
"verifyResourceAccess",
"verifySiteAccess",
"verifyLimits",
"verifyUserHasAction",
"logActionAudit",
"createTarget"
];
const updateTargetStack = [
"verifyTargetAccess",
"verifySiteAccess",
"verifyLimits",
"verifyUserHasAction",
"logActionAudit",
"updateTarget"
];
// Verify verifySiteAccess comes after resource/target access middleware
{
const siteAccessIndex = createTargetStack.indexOf("verifySiteAccess");
const resourceAccessIndex = createTargetStack.indexOf(
"verifyResourceAccess"
);
assertEquals(
siteAccessIndex > resourceAccessIndex,
true,
"verifySiteAccess must come after verifyResourceAccess in create target stack"
);
}
{
const siteAccessIndex = updateTargetStack.indexOf("verifySiteAccess");
const targetAccessIndex =
updateTargetStack.indexOf("verifyTargetAccess");
assertEquals(
siteAccessIndex > targetAccessIndex,
true,
"verifySiteAccess must come after verifyTargetAccess in update target stack"
);
}
// Verify verifySiteAccess comes before the handler
{
const siteAccessIndex = createTargetStack.indexOf("verifySiteAccess");
const handlerIndex = createTargetStack.indexOf("createTarget");
assertEquals(
siteAccessIndex < handlerIndex,
true,
"verifySiteAccess must come before createTarget handler"
);
}
{
const siteAccessIndex = updateTargetStack.indexOf("verifySiteAccess");
const handlerIndex = updateTargetStack.indexOf("updateTarget");
assertEquals(
siteAccessIndex < handlerIndex,
true,
"verifySiteAccess must come before updateTarget handler"
);
}
console.log("All route stack ordering tests passed.");
}
// --- Security scenario tests ---
function testSecurityScenarios() {
console.log("Running security scenario tests...");
// Scenario 1: Attacker has resource access in org_attacker, but tries to
// bind target to a site in org_victim.
// verifyResourceAccess passes (sets req.userOrgId = "org_attacker").
// verifySiteAccess loads site (org_victim), checks site.orgId !== req.userOrgId.
// Expected: 403 Forbidden.
{
const shouldReject = shouldRejectCrossOrgSite(
"org_victim",
"org_attacker"
);
assertEquals(
shouldReject,
true,
"Scenario 1: Cross-org site binding must be rejected"
);
}
// Scenario 2: Attacker has resource access AND site access in another org.
// Even though the user has site access, verifySiteAccess rejects because
// the org-match check runs before the site access check.
// Expected: 403 Forbidden (org mismatch caught before site access check).
{
const shouldReject = shouldRejectCrossOrgSite(
"org_victim",
"org_attacker"
);
assertEquals(
shouldReject,
true,
"Scenario 2: Cross-org site must be rejected even if user has site access"
);
}
// Scenario 3: Legitimate user creates target with site in same org.
// verifyResourceAccess passes, verifySiteAccess org-match passes (same org),
// verifySiteAccess site access passes.
// Expected: 201 Created.
{
const shouldReject = shouldRejectCrossOrgSite(
"org_attacker",
"org_attacker"
);
assertEquals(
shouldReject,
false,
"Scenario 3: Same-org site must be allowed"
);
}
// Scenario 4: WireGuard site in victim org — org mismatch is caught before
// any DB write, pickPort, addPeer, or addTargets side effect.
{
const shouldReject = shouldRejectCrossOrgSite(
"org_victim",
"org_attacker"
);
assertEquals(
shouldReject,
true,
"Scenario 4: WireGuard cross-org site must be rejected before addPeer"
);
}
// Scenario 5: Newt site in victim org — same as scenario 4 but for newt.
{
const shouldReject = shouldRejectCrossOrgSite(
"org_victim",
"org_attacker"
);
assertEquals(
shouldReject,
true,
"Scenario 5: Newt cross-org site must be rejected before addTargets"
);
}
// Scenario 6: Normal site-only route (e.g. PUT /site/:siteId) where
// verifySiteAccess runs without a prior verifyResourceAccess.
// req.userOrgId is undefined, so the org-match check is skipped.
// Normal site access verification proceeds.
{
const shouldReject = shouldRejectCrossOrgSite("org_victim", undefined);
assertEquals(
shouldReject,
false,
"Scenario 6: Site-only routes should skip org-match check"
);
}
console.log("All security scenario tests passed.");
}
// Run all tests
testSiteOrgMatchLogic();
testShouldRejectCrossOrgSite();
testRouteStackOrdering();
testSecurityScenarios();
+11 -6
View File
@@ -71,6 +71,15 @@ export async function verifySiteAccess(
);
}
if (req.userOrgId && site.orgId !== req.userOrgId) {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"User does not have access to this site"
)
);
}
if (!req.userOrg) {
// Get user's role ID in the organization
const userOrgRole = await db
@@ -106,8 +115,7 @@ export async function verifySiteAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
@@ -128,10 +136,7 @@ export async function verifySiteAccess(
.where(
and(
eq(roleSites.siteId, site.siteId),
inArray(
roleSites.roleId,
req.userOrgRoleIds!
)
inArray(roleSites.roleId, req.userOrgRoleIds!)
)
)
.limit(1)
@@ -1,10 +1,16 @@
import { Request, Response, NextFunction } from "express";
import { db, userOrgs, siteProvisioningKeys, siteProvisioningKeyOrg } from "@server/db";
import {
db,
userOrgs,
siteProvisioningKeys,
siteProvisioningKeyOrg
} from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getFirstString } from "@server/lib/requestParams";
export async function verifySiteProvisioningKeyAccess(
req: Request,
@@ -13,8 +19,10 @@ export async function verifySiteProvisioningKeyAccess(
) {
try {
const userId = req.user!.userId;
const siteProvisioningKeyId = req.params.siteProvisioningKeyId;
const orgId = req.params.orgId;
const siteProvisioningKeyId = getFirstString(
req.params.siteProvisioningKeyId
);
const orgId = getFirstString(req.params.orgId);
if (!userId) {
return next(
@@ -80,10 +88,7 @@ export async function verifySiteProvisioningKeyAccess(
.where(
and(
eq(userOrgs.userId, userId),
eq(
userOrgs.orgId,
row.siteProvisioningKeyOrg.orgId
)
eq(userOrgs.orgId, row.siteProvisioningKeyOrg.orgId)
)
)
.limit(1);
@@ -110,8 +115,7 @@ export async function verifySiteProvisioningKeyAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
@@ -103,8 +103,7 @@ export async function verifySiteResourceAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+4 -3
View File
@@ -7,6 +7,7 @@ import HttpCode from "@server/types/HttpCode";
import { canUserAccessResource } from "../auth/canUserAccessResource";
import { checkOrgAccessPolicy } from "#dynamic/lib/checkOrgAccessPolicy";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyTargetAccess(
req: Request,
@@ -14,7 +15,8 @@ export async function verifyTargetAccess(
next: NextFunction
) {
const userId = req.user!.userId;
const targetId = parseInt(req.params.targetId);
const targetIdRaw = getFirstString(req.params.targetId);
const targetId = Number.parseInt(targetIdRaw ?? "", 10);
if (!userId) {
return next(
@@ -120,8 +122,7 @@ export async function verifyTargetAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
+1 -2
View File
@@ -59,8 +59,7 @@ export async function verifyUserAccess(
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Failed organization access policy check: " +
(policyCheck.error || "Unknown error")
"" + (policyCheck.error || "Unknown error")
)
);
}
@@ -38,7 +38,7 @@ export function verifyUserCanSetUserOrgRoles() {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"User does not have permission perform this action"
"User does not have permission to set user organization roles"
)
);
} catch (error) {
@@ -0,0 +1,102 @@
import { Response, NextFunction } from "express";
import { db } from "@server/db";
import { resourceSessions, users } from "@server/db";
import { eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getUserOrgRoleIds } from "@server/lib/userOrgRoles";
import { getUserSessionWithUser } from "@server/db/queries/verifySessionQueries";
import { encodeHexLowerCase } from "@oslojs/encoding";
import { sha256 } from "@oslojs/crypto/sha2";
import config from "@server/lib/config";
import logger from "@server/logger";
export const verifyUserFromResourceSessionMiddleware = async (
req: any,
res: Response,
next: NextFunction
) => {
if (!req.user) {
const sessionCookieName =
config.getRawConfig().server.session_cookie_name;
// Collect all resource session cookies (format: {name}[_s].{timestamp}=token)
const cookieHeader: string | undefined = req.headers.cookie;
const candidates: { timestamp: number; token: string }[] = [];
if (cookieHeader) {
for (const part of cookieHeader.split(";")) {
const trimmed = part.trim();
const eqIdx = trimmed.indexOf("=");
if (eqIdx === -1) continue;
const cookieName = trimmed.slice(0, eqIdx).trim();
const cookieValue = trimmed.slice(eqIdx + 1).trim();
// Match both secure (_s.timestamp) and non-secure (.timestamp) variants
const securePrefix = `${sessionCookieName}_s.`;
const httpPrefix = `${sessionCookieName}.`;
let timestampStr: string | null = null;
if (cookieName.startsWith(securePrefix)) {
timestampStr = cookieName.slice(securePrefix.length);
} else if (cookieName.startsWith(httpPrefix)) {
timestampStr = cookieName.slice(httpPrefix.length);
}
if (timestampStr !== null && /^\d+$/.test(timestampStr)) {
candidates.push({
timestamp: parseInt(timestampStr, 10),
token: cookieValue
});
}
}
}
// Pick the most recently issued session (highest timestamp)
candidates.sort((a, b) => b.timestamp - a.timestamp);
const best = candidates[0];
if (best) {
try {
const sessionId = encodeHexLowerCase(
sha256(new TextEncoder().encode(best.token))
);
const [resourceSession] = await db
.select()
.from(resourceSessions)
.where(eq(resourceSessions.sessionId, sessionId))
.limit(1);
if (resourceSession && Date.now() < resourceSession.expiresAt) {
if (resourceSession.userSessionId) {
const result = await getUserSessionWithUser(
resourceSession.userSessionId
);
if (result?.user && result?.session) {
req.user = result.user;
req.session = result.session;
}
}
}
} catch (e) {
logger.error(
"verifyUserFromResourceSessionMiddleware: failed to validate resource session",
e
);
}
}
}
// Populate userOrgRoleIds if an orgId is available in route params
if (req.user && req.params?.orgId && !req.userOrgRoleIds) {
req.userOrgRoleIds = await getUserOrgRoleIds(
req.user.userId,
req.params.orgId
);
}
next();
};
+2 -1
View File
@@ -4,6 +4,7 @@ import { userOrgs } from "@server/db";
import { and, eq } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { getFirstString } from "@server/lib/requestParams";
export async function verifyUserIsOrgOwner(
req: Request,
@@ -11,7 +12,7 @@ export async function verifyUserIsOrgOwner(
next: NextFunction
) {
const userId = req.user!.userId;
const orgId = req.params.orgId;
const orgId = getFirstString(req.params.orgId);
if (!userId) {
return next(
+1
View File
@@ -7,6 +7,7 @@ export enum OpenAPITags {
Org = "Organization",
PublicResource = "Public Resource",
PrivateResource = "Private Resource",
Policy = "Policy",
Role = "Role",
User = "User",
Invitation = "User Invitation",
+9 -9
View File
@@ -693,9 +693,9 @@ async function syncAcmeCerts(acmeJsonPath: string): Promise<void> {
);
continue;
}
logger.debug(
`acmeCertSync: found ${resolverData.Certificates.length} certificate(s) for resolver "${resolver}"`
);
// logger.debug(
// `acmeCertSync: found ${resolverData.Certificates.length} certificate(s) for resolver "${resolver}"`
// );
for (const cert of resolverData.Certificates) {
allCerts.push(cert);
}
@@ -780,9 +780,9 @@ async function syncAcmeCerts(acmeJsonPath: string): Promise<void> {
}
}
logger.debug(
`acmeCertSync: cert for ${mainDomain} covers ${allDomains.size} domain(s): ${[...allDomains].join(", ")}`
);
// logger.debug(
// `acmeCertSync: cert for ${mainDomain} covers ${allDomains.size} domain(s): ${[...allDomains].join(", ")}`
// );
for (const domain of allDomains) {
try {
@@ -863,9 +863,9 @@ export function initAcmeCertSync(): void {
);
return;
}
logger.debug(
`acmeCertSync: found ${files.length} acme.json file(s) in directory "${acmeJsonPath}"`
);
// logger.debug(
// `acmeCertSync: found ${files.length} acme.json file(s) in directory "${acmeJsonPath}"`
// );
for (const file of files) {
syncAcmeCerts(file).catch((err) => {
logger.error(
+41 -7
View File
@@ -17,7 +17,7 @@ import { certificates, db } from "@server/db";
import { and, eq, isNotNull, or, inArray, sql } from "drizzle-orm";
import { decrypt } from "@server/lib/crypto";
import logger from "@server/logger";
import cache from "#private/lib/cache";
import { regionalCache as cache } from "#private/lib/cache";
import { build } from "@server/build";
// Define the return type for clarity and type safety
@@ -36,8 +36,6 @@ export async function getValidCertificatesForDomains(
domains: Set<string>,
useCache: boolean = true
): Promise<Array<CertificateResult>> {
const finalResults: CertificateResult[] = [];
const domainsToQuery = new Set<string>();
@@ -49,7 +47,26 @@ export async function getValidCertificatesForDomains(
if (cachedCert) {
finalResults.push(cachedCert); // Valid cache hit
} else {
domainsToQuery.add(domain); // Cache miss or expired
// Also check for a wildcard cache entry covering this domain's parent
const parts = domain.split(".");
let wildcardHit = false;
if (parts.length > 1) {
const parentDomain = parts.slice(1).join(".");
const wildcardCacheKey = `cert:*.${parentDomain}`;
const cachedWildcard =
await cache.get<CertificateResult>(wildcardCacheKey);
if (cachedWildcard) {
// Re-stamp queriedDomain so callers see the originally requested domain
finalResults.push({
...cachedWildcard,
queriedDomain: domain
});
wildcardHit = true;
}
}
if (!wildcardHit) {
domainsToQuery.add(domain); // Cache miss or expired
}
}
}
} else {
@@ -59,7 +76,10 @@ export async function getValidCertificatesForDomains(
// 2. If all domains were resolved from the cache, return early
if (domainsToQuery.size === 0) {
const decryptedResults = decryptFinalResults(finalResults, config.getRawConfig().server.secret!);
const decryptedResults = decryptFinalResults(
finalResults,
config.getRawConfig().server.secret!
);
return decryptedResults;
}
@@ -78,7 +98,8 @@ export async function getValidCertificatesForDomains(
const parentDomainsArray = Array.from(parentDomainsToQuery);
// Build wildcard variants: for each parent domain "example.com", also query "*.example.com"
const wildcardPrefixedArray = build != "saas" ? parentDomainsArray.map((d) => `*.${d}`) : [];
const wildcardPrefixedArray =
build != "saas" ? parentDomainsArray.map((d) => `*.${d}`) : [];
// 4. Build and execute a single, efficient Drizzle query
// This query fetches all potential exact and wildcard matches in one database round-trip.
@@ -172,11 +193,24 @@ export async function getValidCertificatesForDomains(
if (useCache) {
const cacheKey = `cert:${domain}`;
await cache.set(cacheKey, resultCert, 180);
// Also cache wildcard certs under a pattern key so other subdomains
// can find them without a DB round-trip
if (resultCert.wildcard) {
const normalizedCertDomain = normalizeWildcardDomain(
resultCert.domain
);
const wildcardCacheKey = `cert:*.${normalizedCertDomain}`;
await cache.set(wildcardCacheKey, resultCert, 180);
}
}
}
}
const decryptedResults = decryptFinalResults(finalResults, config.getRawConfig().server.secret!);
const decryptedResults = decryptFinalResults(
finalResults,
config.getRawConfig().server.secret!
);
return decryptedResults;
}
+78 -10
View File
@@ -21,6 +21,49 @@ import {
} from "@server/lib/checkOrgAccessPolicy";
import { UserType } from "@server/types/UserTypes";
function formatMaxSessionLengthRequirement(
maxSessionLengthHours: number
): string {
if (maxSessionLengthHours < 24) {
return `This organization requires you to log in every ${maxSessionLengthHours} hours.`;
}
const maxDays = Math.round(maxSessionLengthHours / 24);
return `This organization requires you to log in every ${maxDays} days.`;
}
function buildOrgAccessPolicyError(
policies: CheckOrgAccessPolicyResult["policies"]
): string | undefined {
if (!policies) {
return undefined;
}
const errors: string[] = [];
if (policies.requiredTwoFactor === false) {
errors.push(
"This organization requires two-factor authentication. Enable two-factor authentication on your account to continue."
);
}
if (policies.maxSessionLength?.compliant === false) {
errors.push(
`Your session has expired. ${formatMaxSessionLengthRequirement(
policies.maxSessionLength.maxSessionLengthHours
)}`
);
}
if (policies.passwordAge?.compliant === false) {
errors.push(
`Your password has expired. This organization requires you to change your password every ${policies.passwordAge.maxPasswordAgeDays} days.`
);
}
return errors.length > 0 ? errors.join(" ") : undefined;
}
export function enforceResourceSessionLength(
resourceSession: ResourceSession,
org: Org
@@ -36,13 +79,17 @@ export function enforceResourceSessionLength(
if (sessionAgeMs > maxSessionLengthMs) {
return {
valid: false,
error: `Resource session has expired due to organization policy (max session length: ${maxSessionLengthHours} hours)`
error: `Your resource session has expired. ${formatMaxSessionLengthRequirement(
maxSessionLengthHours
)}`
};
}
} else {
return {
valid: false,
error: `Resource session is invalid due to organization policy (max session length: ${maxSessionLengthHours} hours)`
error: `Your resource session is invalid. ${formatMaxSessionLengthRequirement(
maxSessionLengthHours
)}`
};
}
}
@@ -60,14 +107,20 @@ export async function checkOrgAccessPolicy(
if (!orgId) {
return {
allowed: false,
error: "Organization ID is required"
error: "Unable to verify organization access. Organization information is missing."
};
}
if (!userId) {
return { allowed: false, error: "User ID is required" };
return {
allowed: false,
error: "Unable to verify organization access. User information is missing."
};
}
if (!sessionId) {
return { allowed: false, error: "Session ID is required" };
return {
allowed: false,
error: "Your session is invalid. Please log in again."
};
}
if (build === "enterprise") {
@@ -89,7 +142,10 @@ export async function checkOrgAccessPolicy(
.where(eq(orgs.orgId, orgId));
props.org = orgQuery;
if (!props.org) {
return { allowed: false, error: "Organization not found" };
return {
allowed: false,
error: "This organization could not be found."
};
}
}
@@ -100,7 +156,10 @@ export async function checkOrgAccessPolicy(
.where(eq(users.userId, userId));
props.user = userQuery;
if (!props.user) {
return { allowed: false, error: "User not found" };
return {
allowed: false,
error: "Your account could not be found."
};
}
}
@@ -111,14 +170,17 @@ export async function checkOrgAccessPolicy(
.where(eq(sessions.sessionId, sessionId));
props.session = sessionQuery;
if (!props.session) {
return { allowed: false, error: "Session not found" };
return {
allowed: false,
error: "Your session has expired. Please log in again."
};
}
}
if (props.session.userId !== props.user.userId) {
return {
allowed: false,
error: "Session does not belong to the user"
error: "Your session is invalid. Please log in again."
};
}
@@ -187,8 +249,14 @@ export async function checkOrgAccessPolicy(
allowed = false;
}
const policyError = buildOrgAccessPolicyError(policies);
return {
allowed,
policies
policies,
error: allowed
? undefined
: (policyError ??
"You do not meet this organization's security requirements.")
};
}
+81 -4
View File
@@ -18,12 +18,15 @@ import {
resources,
targets,
sites,
siteLabels,
remoteExitNodes,
remoteExitNodePreferenceLabels,
targetHealthCheck,
Transaction
} from "@server/db";
import logger from "@server/logger";
import { ExitNodePingResult } from "@server/routers/newt";
import { eq, and, or, ne, isNull } from "drizzle-orm";
import { eq, and, or, ne, isNull, inArray } from "drizzle-orm";
import axios from "axios";
import config from "../config";
@@ -150,7 +153,8 @@ export async function verifyExitNodeOrgAccess(
export async function listExitNodes(
orgId: string,
filterOnline = false,
noCloud = false
noCloud = false,
siteId?: number
) {
const allExitNodes = await db
.select({
@@ -237,7 +241,7 @@ export async function listExitNodes(
// })
// );
const remoteExitNodes = allExitNodes.filter(
let remoteExitNodesList = allExitNodes.filter(
(node) =>
node.type === "remoteExitNode" && (!filterOnline || node.online)
);
@@ -246,9 +250,82 @@ export async function listExitNodes(
node.type === "gerbil" && (!filterOnline || node.online) && !noCloud
);
// Apply label-based filtering to remote exit nodes if siteId is provided
if (siteId !== undefined && remoteExitNodesList.length > 0) {
// Get the site's labels
const siteLabelRows = await db
.select({ labelId: siteLabels.labelId })
.from(siteLabels)
.where(eq(siteLabels.siteId, siteId));
const siteLabelIds = new Set(siteLabelRows.map((r) => r.labelId));
// Get the remoteExitNode records for these exit nodes so we have the remoteExitNodeId
const exitNodeIds = remoteExitNodesList.map((n) => n.exitNodeId);
const remoteNodeRows = await db
.select({
exitNodeId: remoteExitNodes.exitNodeId,
remoteExitNodeId: remoteExitNodes.remoteExitNodeId
})
.from(remoteExitNodes)
.where(inArray(remoteExitNodes.exitNodeId, exitNodeIds));
const exitNodeIdToRemoteId = new Map(
remoteNodeRows
.filter((r) => r.exitNodeId !== null)
.map((r) => [r.exitNodeId!, r.remoteExitNodeId])
);
// Get preference labels for all remote exit nodes
const remoteExitNodeIds = remoteNodeRows.map((r) => r.remoteExitNodeId);
const prefLabelRows =
remoteExitNodeIds.length > 0
? await db
.select({
remoteExitNodeId:
remoteExitNodePreferenceLabels.remoteExitNodeId,
labelId: remoteExitNodePreferenceLabels.labelId
})
.from(remoteExitNodePreferenceLabels)
.where(
inArray(
remoteExitNodePreferenceLabels.remoteExitNodeId,
remoteExitNodeIds
)
)
: [];
// Build a map of remoteExitNodeId -> Set of labelIds
const prefLabelsMap = new Map<string, Set<number>>();
for (const row of prefLabelRows) {
if (!prefLabelsMap.has(row.remoteExitNodeId)) {
prefLabelsMap.set(row.remoteExitNodeId, new Set());
}
prefLabelsMap.get(row.remoteExitNodeId)!.add(row.labelId);
}
// Filter: include node if it has no preference labels, or if site shares at least one label
const filtered = remoteExitNodesList.filter((node) => {
const remoteId = exitNodeIdToRemoteId.get(node.exitNodeId);
if (!remoteId) return true; // no remoteExitNode record, don't filter
const prefLabels = prefLabelsMap.get(remoteId);
if (!prefLabels || prefLabels.size === 0) return true; // no preference labels, include
// include only if site has at least one matching label
for (const labelId of siteLabelIds) {
if (prefLabels.has(labelId)) return true;
}
return false;
});
// Only apply the filtered list if at least one remote node remains;
// otherwise fall through to the gerbil fallback below
if (filtered.length > 0 || remoteExitNodesList.length === 0) {
remoteExitNodesList = filtered;
}
}
// THIS PROVIDES THE FALL
const exitNodesList =
remoteExitNodes.length > 0 ? remoteExitNodes : gerbilExitNodes;
remoteExitNodesList.length > 0 ? remoteExitNodesList : gerbilExitNodes;
return exitNodesList;
}
+140 -84
View File
@@ -11,37 +11,80 @@
* This file is not licensed under the AGPLv3.
*/
import { config } from "@server/lib/config";
import logger from "@server/logger";
import { redis } from "#private/lib/redis";
import { v4 as uuidv4 } from "uuid";
const instanceId = uuidv4();
type LocalLockRecord = {
owner: string;
expiresAt: number;
};
const localLocks = new Map<string, LocalLockRecord>();
export class LockManager {
private clearExpiredLocalLock(lockKey: string): void {
const current = localLocks.get(lockKey);
if (current && current.expiresAt <= Date.now()) {
localLocks.delete(lockKey);
}
}
/**
* Acquire a distributed lock using Redis SET with NX and PX options
* @param lockKey - Unique identifier for the lock
* @param ttlMs - Time to live in milliseconds
* @returns Promise<boolean> - true if lock acquired, false otherwise
* @returns Promise<string | null> - a token identifying this specific acquisition
* (truthy) on success, or null if the lock could not be acquired.
*/
async acquireLock(
lockKey: string,
ttlMs: number = 30000,
maxRetries: number = 3,
retryDelayMs: number = 100
): Promise<boolean> {
): Promise<string | null> {
if (!redis || !redis.status || redis.status !== "ready") {
return true;
for (let attempt = 0; attempt < maxRetries; attempt++) {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
const token = `${instanceId}:${uuidv4()}`;
localLocks.set(lockKey, {
owner: token,
expiresAt: Date.now() + ttlMs
});
return token;
}
// Do not treat a same-process holder as automatically
// reentrant -- see the note in the Redis branch below.
if (attempt < maxRetries - 1) {
const delay = retryDelayMs * Math.pow(2, attempt);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
return null;
}
const lockValue = `${
instanceId
}:${Date.now()}`;
const redisKey = `lock:${lockKey}`;
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
// Every acquisition attempt gets its own unique token, even
// within the same process. Two independent logical operations
// (e.g. two different API requests handled by the same server)
// racing for this key must never both believe they hold the
// lock -- if we treated "existing value starts with my
// instanceId" as reentrant success, a second unrelated caller
// on this process could barge in while the first is still
// mid-flight, and their writes under the lock would interleave
// unguarded.
const lockValue = `${instanceId}:${uuidv4()}`;
// Use SET with NX (only set if not exists) and PX (expire in milliseconds)
// This is atomic and handles both setting and expiration
const result = await redis.set(
@@ -53,30 +96,8 @@ export class LockManager {
);
if (result === "OK") {
logger.debug(
`Lock acquired: ${lockKey} by ${
instanceId
}`
);
return true;
}
// Check if the existing lock is from this worker (reentrant behavior)
const existingValue = await redis.get(redisKey);
if (
existingValue &&
existingValue.startsWith(
`${instanceId}:`
)
) {
// Extend the lock TTL since it's the same worker
await redis.pexpire(redisKey, ttlMs);
logger.debug(
`Lock extended: ${lockKey} by ${
instanceId
}`
);
return true;
logger.debug(`Lock acquired: ${lockKey} by ${instanceId}`);
return lockValue;
}
// If this isn't our last attempt, wait before retrying with exponential backoff
@@ -88,7 +109,10 @@ export class LockManager {
await new Promise((resolve) => setTimeout(resolve, delay));
}
} catch (error) {
logger.error(`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`, error);
logger.error(
`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`,
error
);
// On error, still retry if we have attempts left
if (attempt < maxRetries - 1) {
const delay = retryDelayMs * Math.pow(2, attempt);
@@ -100,27 +124,36 @@ export class LockManager {
logger.debug(
`Failed to acquire lock ${lockKey} after ${maxRetries} attempts`
);
return false;
return null;
}
/**
* Release a lock using Lua script to ensure atomicity
* Release a lock previously acquired via acquireLock/acquireLockWithRetry,
* using a Lua script to ensure we only delete it if it still matches the
* exact token from that acquisition (not just "owned by this process") --
* this ensures a caller whose TTL already expired can't delete a
* different, currently-active holder's lock.
* @param lockKey - Unique identifier for the lock
* @param token - the exact token returned by the acquisition being released
*/
async releaseLock(lockKey: string): Promise<void> {
async releaseLock(lockKey: string, token: string): Promise<void> {
if (!redis || !redis.status || redis.status !== "ready") {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (existing && existing.owner === token) {
localLocks.delete(lockKey);
}
return;
}
const redisKey = `lock:${lockKey}`;
// Lua script to ensure we only delete the lock if it belongs to this worker
const luaScript = `
local key = KEYS[1]
local worker_prefix = ARGV[1]
local expected_value = ARGV[1]
local current_value = redis.call('GET', key)
if current_value and string.find(current_value, worker_prefix, 1, true) == 1 then
if current_value and current_value == expected_value then
return redis.call('DEL', key)
else
return 0
@@ -132,20 +165,14 @@ export class LockManager {
luaScript,
1,
redisKey,
`${instanceId}:`
token
)) as number;
if (result === 1) {
logger.debug(
`Lock released: ${lockKey} by ${
instanceId
}`
);
logger.debug(`Lock released: ${lockKey} by ${instanceId}`);
} else {
logger.warn(
`Lock not released - not owned by worker: ${lockKey} by ${
instanceId
}`
`Lock not released - token did not match current holder: ${lockKey} (attempted by ${instanceId})`
);
}
} catch (error) {
@@ -159,6 +186,7 @@ export class LockManager {
*/
async forceReleaseLock(lockKey: string): Promise<void> {
if (!redis || !redis.status || redis.status !== "ready") {
localLocks.delete(lockKey);
return;
}
@@ -186,7 +214,20 @@ export class LockManager {
owner?: string;
}> {
if (!redis || !redis.status || redis.status !== "ready") {
return { exists: false, ownedByMe: true, ttl: 0 };
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
return { exists: false, ownedByMe: false, ttl: 0 };
}
const ttl = Math.max(0, existing.expiresAt - Date.now());
return {
exists: true,
ownedByMe: existing.owner.startsWith(`${instanceId}:`),
ttl,
owner: existing.owner.split(":")[0]
};
}
const redisKey = `lock:${lockKey}`;
@@ -198,11 +239,7 @@ export class LockManager {
]);
const exists = value !== null;
const ownedByMe =
exists &&
value!.startsWith(
`${instanceId}:`
);
const ownedByMe = exists && value!.startsWith(`${instanceId}:`);
const owner = exists ? value!.split(":")[0] : undefined;
return {
@@ -218,26 +255,40 @@ export class LockManager {
}
/**
* Extend the TTL of an existing lock owned by this worker
* Extend the TTL of an existing lock, provided the token matches the
* acquisition currently holding it.
* @param lockKey - Unique identifier for the lock
* @param ttlMs - New TTL in milliseconds
* @param token - the token returned by the acquisition being extended
* @returns Promise<boolean> - true if extended successfully
*/
async extendLock(lockKey: string, ttlMs: number): Promise<boolean> {
async extendLock(
lockKey: string,
ttlMs: number,
token: string
): Promise<boolean> {
if (!redis || !redis.status || redis.status !== "ready") {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing || existing.owner !== token) {
return false;
}
existing.expiresAt = Date.now() + ttlMs;
localLocks.set(lockKey, existing);
return true;
}
const redisKey = `lock:${lockKey}`;
// Lua script to extend TTL only if lock is owned by this worker
const luaScript = `
local key = KEYS[1]
local worker_prefix = ARGV[1]
local expected_value = ARGV[1]
local ttl = tonumber(ARGV[2])
local current_value = redis.call('GET', key)
if current_value and string.find(current_value, worker_prefix, 1, true) == 1 then
if current_value and current_value == expected_value then
return redis.call('PEXPIRE', key, ttl)
else
return 0
@@ -249,15 +300,13 @@ export class LockManager {
luaScript,
1,
redisKey,
`${instanceId}:`,
token,
ttlMs.toString()
)) as number;
if (result === 1) {
logger.debug(
`Lock extended: ${lockKey} by ${
instanceId
} for ${ttlMs}ms`
`Lock extended: ${lockKey} by ${instanceId} for ${ttlMs}ms`
);
return true;
}
@@ -274,23 +323,24 @@ export class LockManager {
* @param ttlMs - Time to live in milliseconds
* @param maxRetries - Maximum number of retry attempts
* @param baseDelayMs - Base delay between retries in milliseconds
* @returns Promise<boolean> - true if lock acquired
* @returns Promise<string | null> - token if acquired, null otherwise
*/
async acquireLockWithRetry(
lockKey: string,
ttlMs: number = 30000,
maxRetries: number = 5,
baseDelayMs: number = 100
): Promise<boolean> {
if (!redis || !redis.status || redis.status !== "ready") {
return true;
}
): Promise<string | null> {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
const acquired = await this.acquireLock(lockKey, ttlMs);
const acquired = await this.acquireLock(
lockKey,
ttlMs,
1,
baseDelayMs
);
if (acquired) {
return true;
return acquired;
}
if (attempt < maxRetries) {
@@ -304,7 +354,7 @@ export class LockManager {
logger.warn(
`Failed to acquire lock ${lockKey} after ${maxRetries + 1} attempts`
);
return false;
return null;
}
/**
@@ -319,20 +369,16 @@ export class LockManager {
fn: () => Promise<T>,
ttlMs: number = 30000
): Promise<T> {
if (!redis || !redis.status || redis.status !== "ready") {
return await fn();
}
const token = await this.acquireLock(lockKey, ttlMs);
const acquired = await this.acquireLock(lockKey, ttlMs);
if (!acquired) {
if (!token) {
throw new Error(`Failed to acquire lock: ${lockKey}`);
}
try {
return await fn();
} finally {
await this.releaseLock(lockKey);
await this.releaseLock(lockKey, token);
}
}
@@ -346,7 +392,21 @@ export class LockManager {
locksOwnedByMe: number;
}> {
if (!redis || !redis.status || redis.status !== "ready") {
return { activeLocksCount: 0, locksOwnedByMe: 0 };
const now = Date.now();
for (const [key, value] of localLocks.entries()) {
if (value.expiresAt <= now) {
localLocks.delete(key);
}
}
let locksOwnedByMe = 0;
for (const value of localLocks.values()) {
if (value.owner.startsWith(`${instanceId}:`)) {
locksOwnedByMe++;
}
}
return { activeLocksCount: localLocks.size, locksOwnedByMe };
}
try {
@@ -356,11 +416,7 @@ export class LockManager {
if (keys.length > 0) {
const values = await redis.mget(...keys);
locksOwnedByMe = values.filter(
(value) =>
value &&
value.startsWith(
`${instanceId}:`
)
(value) => value && value.startsWith(`${instanceId}:`)
).length;
}
+105
View File
@@ -0,0 +1,105 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import { redis } from "#private/lib/redis";
import logger from "@server/logger";
export const ORG_REBUILD_CONCURRENCY_LIMIT = 10;
// Safety-net TTL: slightly longer than the rebuild lock TTL (120 s). If a
// server process dies while holding a rebuild, this ensures the counter key
// eventually expires rather than staying inflated forever.
const ORG_REBUILD_COUNT_TTL_MS = 180000;
const KEY_PREFIX = "rebuild-org-count:";
// In-memory fallback used when Redis is unavailable.
const localFallback = new Map<string, number>();
function isRedisReady(): boolean {
return !!(redis && redis.status === "ready");
}
export async function incrementOrgRebuildCount(orgId: string): Promise<void> {
if (!isRedisReady()) {
localFallback.set(orgId, (localFallback.get(orgId) ?? 0) + 1);
return;
}
try {
const key = `${KEY_PREFIX}${orgId}`;
await redis!.incr(key);
// Always refresh the TTL so the key doesn't expire while rebuilds are
// still in progress. The TTL is purely a crash-recovery safety net.
await redis!.pexpire(key, ORG_REBUILD_COUNT_TTL_MS);
} catch (err) {
logger.warn(
`orgRebuildCounter: Redis increment failed for org ${orgId}, falling back to local:`,
err
);
localFallback.set(orgId, (localFallback.get(orgId) ?? 0) + 1);
}
}
export async function decrementOrgRebuildCount(orgId: string): Promise<void> {
if (!isRedisReady()) {
const current = localFallback.get(orgId) ?? 0;
if (current <= 1) {
localFallback.delete(orgId);
} else {
localFallback.set(orgId, current - 1);
}
return;
}
try {
const key = `${KEY_PREFIX}${orgId}`;
const count = await redis!.decr(key);
if (count <= 0) {
await redis!.del(key);
}
} catch (err) {
logger.warn(
`orgRebuildCounter: Redis decrement failed for org ${orgId}, falling back to local:`,
err
);
const current = localFallback.get(orgId) ?? 0;
if (current <= 1) {
localFallback.delete(orgId);
} else {
localFallback.set(orgId, current - 1);
}
}
}
export async function getOrgActiveRebuildCount(orgId: string): Promise<number> {
if (!isRedisReady()) {
return localFallback.get(orgId) ?? 0;
}
try {
const key = `${KEY_PREFIX}${orgId}`;
const val = await redis!.get(key);
return val ? parseInt(val, 10) : 0;
} catch (err) {
logger.warn(
`orgRebuildCounter: Redis get failed for org ${orgId}, falling back to local:`,
err
);
return localFallback.get(orgId) ?? 0;
}
}
export async function checkOrgRebuildRateLimit(
orgId: string
): Promise<boolean> {
return (
(await getOrgActiveRebuildCount(orgId)) >= ORG_REBUILD_CONCURRENCY_LIMIT
);
}
+16 -18
View File
@@ -12,7 +12,7 @@
*/
import logger from "@server/logger";
import redisManager from "#private/lib/redis";
import { regionalRedisManager as redisManager } from "#private/lib/redis";
import { build } from "@server/build";
// Rate limiting configuration
@@ -152,10 +152,9 @@ export class RateLimitService {
);
// Set TTL using the client directly - this prevents the key from persisting forever
if (redisManager.getClient()) {
await redisManager
.getClient()
.expire(globalKey, RATE_LIMIT_WINDOW + 10);
const writeClient = redisManager.getClient();
if (writeClient) {
await writeClient.expire(globalKey, RATE_LIMIT_WINDOW + 10);
}
// Update tracking
@@ -204,10 +203,12 @@ export class RateLimitService {
);
// Set TTL using the client directly - this prevents the key from persisting forever
if (redisManager.getClient()) {
await redisManager
.getClient()
.expire(messageTypeKey, RATE_LIMIT_WINDOW + 10);
const writeClient = redisManager.getClient();
if (writeClient) {
await writeClient.expire(
messageTypeKey,
RATE_LIMIT_WINDOW + 10
);
}
// Update tracking
@@ -487,16 +488,13 @@ export class RateLimitService {
await redisManager.del(globalKey);
// Get all message type keys for this client and delete them
const client = redisManager.getClient();
if (client) {
const messageTypeKeys = await client.keys(
`ratelimit:${clientId}:*`
const messageTypeKeys = await redisManager.keys(
`ratelimit:${clientId}:*`
);
if (messageTypeKeys.length > 0) {
await Promise.all(
messageTypeKeys.map((key) => redisManager.del(key))
);
if (messageTypeKeys.length > 0) {
await Promise.all(
messageTypeKeys.map((key) => redisManager.del(key))
);
}
}
}
}
+5 -1
View File
@@ -109,7 +109,11 @@ export const privateConfigSchema = z
enable_redis: z.boolean().optional().default(false),
use_pangolin_dns: z.boolean().optional().default(false),
use_org_only_idp: z.boolean().optional(),
enable_acme_cert_sync: z.boolean().optional().default(true)
enable_acme_cert_sync: z.boolean().optional().default(true),
disable_private_http_placeholder: z
.boolean()
.optional()
.default(false)
})
.optional()
.prefault({}),
+241
View File
@@ -0,0 +1,241 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import { redis } from "#private/lib/redis";
import { lockManager } from "#private/lib/lock";
import logger from "@server/logger";
import { isTransientError } from "@server/lib/dbRetry";
export type RebuildJobType = "site-resource" | "client";
export interface RebuildJob {
type: RebuildJobType;
id: number;
// Number of times this job has already been re-queued after a transient
// failure. Absent/0 means it has not failed yet.
attempt?: number;
}
export interface RebuildJobHandlers {
onSiteResource(siteResourceId: number): Promise<void>;
onClient(clientId: number): Promise<void>;
}
// Redis list holding pending rebuild jobs (RPUSH to enqueue, LPOP to dequeue — FIFO order).
const QUEUE_KEY = "rebuild-client-associations:queue";
const QUEUED_SET_KEY = "rebuild-client-associations:queued";
// Distributed lock that serialises queue consumption to a single server instance
// at a time. TTL is generous enough to cover a full batch of expensive rebuilds.
const PROCESSOR_LOCK_KEY = "rebuild-client-associations:processor";
// Each rebuild can take up to REBUILD_ASSOCIATIONS_LOCK_TTL_MS (120 s) per
// resource. Allow BATCH_SIZE resources per processor-lock acquisition, plus a
// small buffer.
const BATCH_SIZE = 5;
const PROCESSOR_LOCK_TTL_MS = 120000 * BATCH_SIZE + 30000; // ~630 s
const POLL_INTERVAL_MS = 500;
// A job that fails with a transient DB error gets re-queued with backoff
// instead of being dropped, up to this many times.
const MAX_JOB_ATTEMPTS = 5;
const JOB_RETRY_BASE_DELAY_MS = 1000;
class RedisRebuildQueue {
private processingStarted = false;
async isQueued(job: RebuildJob): Promise<boolean> {
if (!redis || redis.status !== "ready") return false;
const dedupeKey = `${job.type}:${job.id}`;
try {
const member = await redis.sismember(QUEUED_SET_KEY, dedupeKey);
return member === 1;
} catch {
return false;
}
}
async enqueue(job: RebuildJob): Promise<void> {
if (!redis || redis.status !== "ready") {
logger.warn(
`Rebuild queue: Redis not available — rebuild for ${job.type}:${job.id} will not be retried`
);
return;
}
try {
const dedupeKey = `${job.type}:${job.id}`;
const added = await redis.sadd(QUEUED_SET_KEY, dedupeKey);
if (added === 0) {
logger.debug(
`Rebuild queue: skipped duplicate queued job ${job.type}:${job.id}`
);
return;
}
await redis.rpush(QUEUE_KEY, JSON.stringify(job));
logger.debug(
`Rebuild queue: enqueued ${job.type}:${job.id} (queue position: tail)`
);
} catch (err) {
await redis
.srem(QUEUED_SET_KEY, `${job.type}:${job.id}`)
.catch((cleanupErr) =>
logger.warn(
`Rebuild queue: failed to cleanup dedupe key for ${job.type}:${job.id} after enqueue failure:`,
cleanupErr
)
);
logger.error(
`Rebuild queue: failed to enqueue ${job.type}:${job.id}:`,
err
);
}
}
startProcessing(handlers: RebuildJobHandlers): void {
if (this.processingStarted) return;
this.processingStarted = true;
this.processLoop(handlers).catch((err) => {
logger.error("Rebuild queue processor loop crashed:", err);
});
logger.info("Rebuild queue processor started");
}
private async processLoop(handlers: RebuildJobHandlers): Promise<void> {
while (true) {
try {
await this.tryProcessBatch(handlers);
} catch (err) {
logger.error(
"Rebuild queue: unhandled error in process loop:",
err
);
}
await new Promise((resolve) =>
setTimeout(resolve, POLL_INTERVAL_MS)
);
}
}
private async tryProcessBatch(handlers: RebuildJobHandlers): Promise<void> {
if (!redis || redis.status !== "ready") return;
// Peek before acquiring the processor lock to avoid unnecessary Redis
// round-trips and lock contention when the queue is idle.
const queueLength = await redis.llen(QUEUE_KEY).catch(() => 0);
if (queueLength === 0) return;
try {
await lockManager.withLock(
PROCESSOR_LOCK_KEY,
async () => {
for (let i = 0; i < BATCH_SIZE; i++) {
if (!redis || redis.status !== "ready") break;
const payload = await redis.lpop(QUEUE_KEY);
if (payload === null) break; // queue drained
let job: RebuildJob;
try {
job = JSON.parse(payload) as RebuildJob;
} catch {
logger.error(
`Rebuild queue: could not parse job payload, discarding: ${payload}`
);
continue;
}
// Remove from dedupe set once dequeued so the same job
// can be re-queued while this one is in progress.
await redis
.srem(QUEUED_SET_KEY, `${job.type}:${job.id}`)
.catch((cleanupErr) =>
logger.warn(
`Rebuild queue: failed to remove dedupe key for ${job.type}:${job.id} on dequeue:`,
cleanupErr
)
);
logger.debug(
`Rebuild queue: processing ${job.type}:${job.id}`
);
try {
if (job.type === "site-resource") {
await handlers.onSiteResource(job.id);
} else if (job.type === "client") {
await handlers.onClient(job.id);
} else {
logger.warn(
`Rebuild queue: unknown job type "${(job as any).type}", discarding`
);
}
logger.debug(
`Rebuild queue: completed ${job.type}:${job.id}`
);
} catch (err) {
const attempt = (job.attempt ?? 0) + 1;
if (
isTransientError(err) &&
attempt <= MAX_JOB_ATTEMPTS
) {
const delay =
JOB_RETRY_BASE_DELAY_MS *
Math.pow(2, attempt - 1);
logger.warn(
`Rebuild queue: job ${job.type}:${job.id} hit a transient error (attempt ${attempt}/${MAX_JOB_ATTEMPTS}), re-queuing in ${delay}ms:`,
err
);
setTimeout(() => {
this.enqueue({ ...job, attempt }).catch(
(enqueueErr) =>
logger.error(
`Rebuild queue: failed to re-queue ${job.type}:${job.id} after transient error:`,
enqueueErr
)
);
}, delay);
} else {
logger.error(
`Rebuild queue: job ${job.type}:${job.id} threw an error:`,
err
);
}
}
}
},
PROCESSOR_LOCK_TTL_MS
);
} catch (err: any) {
if (
typeof err?.message === "string" &&
err.message.startsWith("Failed to acquire lock")
) {
// Another server instance currently holds the processor lock and
// is consuming the queue — nothing to do this cycle.
logger.debug(
"Rebuild queue: processor lock held by another instance, skipping this cycle"
);
} else {
throw err;
}
}
}
}
export const rebuildQueue: RedisRebuildQueue = new RedisRebuildQueue();
+80 -18
View File
@@ -894,6 +894,19 @@ class RegionalRedisManager {
return opts;
}
// The regional Redis StatefulSet's "redis" service pins to pod redis-0
// (primary). The replica (redis-1) is only reachable through the
// per-pod headless service: <svc>.<namespace>.svc.cluster.local ->
// redis-1.redis-headless.<namespace>.svc.cluster.local. Returns null
// if the configured host doesn't match that pattern (e.g. local dev),
// in which case callers should fall back to the primary for reads.
private getReplicaHost(primaryHost: string): string | null {
const match = primaryHost.match(/^redis\.([^.]+)\.svc\.cluster\.local$/);
if (!match) return null;
const namespace = match[1];
return `redis-1.redis-headless.${namespace}.svc.cluster.local`;
}
private initializeClients(): void {
const cfg = this.getConfig();
const baseOpts = {
@@ -907,35 +920,42 @@ class RegionalRedisManager {
try {
this.writeClient = new Redis(baseOpts);
// redis-1 (replica) handles reads; fall back to primary if not resolvable
this.readClient = new Redis({
...baseOpts,
host: cfg.host!.replace(/^(.*?)(\.\S+)$/, (_, h, rest) => {
// Derive replica hostname from the headless service pattern:
// redis.redis.svc.cluster.local -> redis-1.redis-headless.redis.svc.cluster.local
// If it doesn't look like a k8s service, just use the same host
return h + rest;
})
});
// For simplicity use same host for both; callers can always read from primary
// The real replica routing is handled by the StatefulSet headless service
this.readClient = this.writeClient;
const replicaHost = this.getReplicaHost(cfg.host!);
this.readClient = replicaHost
? new Redis({ ...baseOpts, host: replicaHost })
: this.writeClient;
this.writeClient.on("ready", () => {
logger.info("Regional Redis client ready");
logger.info("Regional Redis write client ready");
this.isHealthy = true;
});
this.writeClient.on("error", (err) => {
logger.error("Regional Redis client error:", err);
logger.error("Regional Redis write client error:", err);
this.isHealthy = false;
});
this.writeClient.on("reconnecting", () => {
logger.info("Regional Redis client reconnecting...");
logger.info("Regional Redis write client reconnecting...");
this.isHealthy = false;
});
logger.info("Regional Redis client initialized");
if (this.readClient !== this.writeClient) {
this.readClient.on("ready", () => {
logger.info("Regional Redis read client ready");
});
this.readClient.on("error", (err) => {
logger.error("Regional Redis read client error:", err);
});
this.readClient.on("reconnecting", () => {
logger.info("Regional Redis read client reconnecting...");
});
}
logger.info(
replicaHost
? `Regional Redis client initialized (reads routed to replica ${replicaHost})`
: "Regional Redis client initialized (no replica resolvable, reads routed to primary)"
);
} catch (error) {
logger.error("Failed to initialize regional Redis client:", error);
this.isEnabled = false;
@@ -1000,13 +1020,55 @@ class RegionalRedisManager {
}
}
public getClient(): Redis | null {
return this.writeClient;
}
public async hget(key: string, field: string): Promise<string | null> {
if (!this.isRedisEnabled() || !this.readClient) return null;
try {
return await this.readClient.hget(key, field);
} catch (error) {
logger.error("Regional Redis HGET error:", error);
return null;
}
}
public async hset(
key: string,
field: string,
value: string
): Promise<boolean> {
if (!this.isRedisEnabled() || !this.writeClient) return false;
try {
await this.writeClient.hset(key, field, value);
return true;
} catch (error) {
logger.error("Regional Redis HSET error:", error);
return false;
}
}
public async hgetall(key: string): Promise<Record<string, string>> {
if (!this.isRedisEnabled() || !this.readClient) return {};
try {
return await this.readClient.hgetall(key);
} catch (error) {
logger.error("Regional Redis HGETALL error:", error);
return {};
}
}
public async disconnect(): Promise<void> {
try {
if (this.readClient && this.readClient !== this.writeClient) {
await this.readClient.quit();
}
this.readClient = null;
if (this.writeClient) {
await this.writeClient.quit();
this.writeClient = null;
}
this.readClient = null;
logger.info("Regional Redis client disconnected");
} catch (error) {
logger.error("Error disconnecting regional Redis client:", error);
+428 -31
View File
@@ -84,7 +84,8 @@ export async function getTraefikConfig(
filterOutNamespaceDomains = false,
generateLoginPageRouters = false,
allowRawResources = true,
allowMaintenancePage = true
maintenancePageUiUrl: string | null = null,
browserGatewayUiUrl: string | null = null
): Promise<any> {
// Get resources with their targets and sites in a single optimized query
// Start from sites on this exit node, then join to targets and resources
@@ -95,9 +96,7 @@ export async function getTraefikConfig(
resourceName: resources.name,
fullDomain: resources.fullDomain,
ssl: resources.ssl,
http: resources.http,
proxyPort: resources.proxyPort,
protocol: resources.protocol,
subdomain: resources.subdomain,
domainId: resources.domainId,
enabled: resources.enabled,
@@ -110,6 +109,7 @@ export async function getTraefikConfig(
proxyProtocol: resources.proxyProtocol,
proxyProtocolVersion: resources.proxyProtocolVersion,
wildcard: resources.wildcard,
mode: resources.mode,
maintenanceModeEnabled: resources.maintenanceModeEnabled,
maintenanceModeType: resources.maintenanceModeType,
@@ -172,8 +172,15 @@ export async function getTraefikConfig(
),
inArray(sites.type, siteTypes),
allowRawResources
? isNotNull(resources.http) // ignore the http check if allow_raw_resources is true
: eq(resources.http, true)
? inArray(resources.mode, [
"http",
"udp",
"tcp",
"vnc",
"ssh",
"rdp"
]) // allow all three
: inArray(resources.mode, ["http", "vnc", "ssh", "rdp"])
)
)
.orderBy(desc(targets.priority), targets.targetId); // stable ordering
@@ -181,7 +188,10 @@ export async function getTraefikConfig(
// Group by resource and include targets with their unique site data
const resourcesMap = new Map();
resourcesWithTargetsAndSites.forEach((row) => {
for (const row of resourcesWithTargetsAndSites) {
if (!["http", "tcp", "udp"].includes(row.mode)) {
continue;
}
const resourceId = row.resourceId;
const resourceName = sanitize(row.resourceName) || "";
const targetPath = encodePath(row.path); // Use encodePath to avoid collisions (e.g. "/a/b" vs "/a-b")
@@ -191,7 +201,7 @@ export async function getTraefikConfig(
const priority = row.priority ?? 100;
if (filterOutNamespaceDomains && row.domainNamespaceId) {
return;
continue;
}
// Create a unique key combining resourceId, path config, and rewrite config
@@ -218,7 +228,7 @@ export async function getTraefikConfig(
logger.debug(
`Invalid path rewrite configuration for resource ${resourceId}: ${validation.error}`
);
return;
continue;
}
resourcesMap.set(mapKey, {
@@ -227,9 +237,8 @@ export async function getTraefikConfig(
key: key,
fullDomain: row.fullDomain,
ssl: row.ssl,
http: row.http,
proxyPort: row.proxyPort,
protocol: row.protocol,
mode: row.mode,
subdomain: row.subdomain,
domainId: row.domainId,
enabled: row.enabled,
@@ -277,14 +286,88 @@ export async function getTraefikConfig(
online: row.siteOnline
}
});
});
}
// Group browser gateway targets by resource
type BrowserGatewayResourceEntry = {
resourceId: number;
name: string;
fullDomain: string | null;
ssl: boolean | null;
subdomain: string | null;
domainId: string | null;
enabled: boolean | null;
wildcard: boolean | null;
domainCertResolver: string | null;
preferWildcardCert: boolean | null;
maintenanceModeEnabled: boolean | null;
maintenanceModeType: string | null;
maintenanceTitle: string | null;
maintenanceMessage: string | null;
maintenanceEstimatedTime: string | null;
targets: {
targetId: number;
bgType: string;
siteId: number;
siteType: string;
siteOnline: boolean | null;
subnet: string | null;
}[];
};
const browserGatewayResourcesMap = new Map<
number,
BrowserGatewayResourceEntry
>();
if (browserGatewayUiUrl) {
for (const row of resourcesWithTargetsAndSites) {
if (!["ssh", "vnc", "rdp"].includes(row.mode)) {
continue;
}
if (filterOutNamespaceDomains && row.domainNamespaceId) {
continue;
}
if (!browserGatewayResourcesMap.has(row.resourceId)) {
browserGatewayResourcesMap.set(row.resourceId, {
resourceId: row.resourceId,
name: sanitize(row.resourceName) || "",
fullDomain: row.fullDomain,
ssl: row.ssl,
subdomain: row.subdomain,
domainId: row.domainId,
enabled: row.enabled,
wildcard: row.wildcard,
domainCertResolver: row.domainCertResolver,
preferWildcardCert: row.preferWildcardCert,
maintenanceModeEnabled: row.maintenanceModeEnabled,
maintenanceModeType: row.maintenanceModeType,
maintenanceTitle: row.maintenanceTitle,
maintenanceMessage: row.maintenanceMessage,
maintenanceEstimatedTime: row.maintenanceEstimatedTime,
targets: []
});
}
browserGatewayResourcesMap.get(row.resourceId)!.targets.push({
targetId: row.targetId,
bgType: row.mode,
siteId: row.siteId,
siteType: row.siteType,
siteOnline: row.siteOnline,
subnet: row.subnet
});
}
}
let siteResourcesWithFullDomain: {
siteResourceId: number;
fullDomain: string | null;
mode: "http" | "host" | "cidr";
mode: "http" | "host" | "cidr" | "ssh";
}[] = [];
if (build == "enterprise") {
if (
build == "enterprise" &&
!privateConfig.getRawPrivateConfig().flags
.disable_private_http_placeholder
) {
// we dont want to do this on the cloud
// Query siteResources in HTTP mode with SSL enabled and aliases - cert generation / HTTPS edge
siteResourcesWithFullDomain = await db
@@ -326,6 +409,12 @@ export async function getTraefikConfig(
domains.add(sr.fullDomain);
}
}
// Include browser gateway resource domains
for (const bgResource of browserGatewayResourcesMap.values()) {
if (bgResource.enabled && bgResource.ssl && bgResource.fullDomain) {
domains.add(bgResource.fullDomain);
}
}
// get the valid certs for these domains
validCerts = await getValidCertificatesForDomains(domains, true); // we are caching here because this is called often
// logger.debug(`Valid certs for domains: ${JSON.stringify(validCerts)}`);
@@ -361,16 +450,29 @@ export async function getTraefikConfig(
const transportName = `${key}-transport`;
const headersMiddlewareName = `${key}-headers-middleware`;
logger.debug(
`Processing resource ${resource.name} with domain ${fullDomain} and ${targets.length} targets`
);
if (!resource.enabled) {
logger.debug(
`Resource ${resource.name} is disabled, skipping Traefik config`
);
continue;
}
if (resource.http) {
if (resource.mode == "http") {
if (!resource.domainId) {
logger.debug(
`Resource ${resource.name} does not have a domainId, skipping Traefik config`
);
continue;
}
if (!resource.fullDomain) {
logger.debug(
`Resource ${resource.name} does not have a fullDomain, skipping Traefik config`
);
continue;
}
@@ -530,10 +632,11 @@ export async function getTraefikConfig(
}
}
if (showMaintenancePage && allowMaintenancePage) {
if (showMaintenancePage && maintenancePageUiUrl) {
const maintenanceServiceName = `${key}-maintenance-service`;
const maintenanceRouterName = `${key}-maintenance-router`;
const rewriteMiddlewareName = `${key}-maintenance-rewrite`;
const maintenanceHeadersMiddlewareName = `${key}-maintenance-headers`;
const entrypointHttp =
config.getRawConfig().traefik.http_entrypoint;
@@ -546,15 +649,11 @@ export async function getTraefikConfig(
? `*.${domainParts.slice(1).join(".")}`
: fullDomain;
const maintenancePort = config.getRawConfig().server.next_port;
const maintenanceHost =
config.getRawConfig().server.internal_hostname;
config_output.http.services[maintenanceServiceName] = {
loadBalancer: {
servers: [
{
url: `http://${maintenanceHost}:${maintenancePort}`
url: maintenancePageUiUrl
}
],
passHostHeader: true
@@ -573,12 +672,26 @@ export async function getTraefikConfig(
}
};
config_output.http.middlewares[
maintenanceHeadersMiddlewareName
] = {
headers: {
customRequestHeaders: {
Host: "app.pangolin.net", // if we are sending to the cloud the host needs to be this but we will pull the p-host to find the resource
"p-host": fullDomain
}
}
};
config_output.http.routers[maintenanceRouterName] = {
entryPoints: [
resource.ssl ? entrypointHttps : entrypointHttp
],
service: maintenanceServiceName,
middlewares: [rewriteMiddlewareName],
middlewares: [
rewriteMiddlewareName,
maintenanceHeadersMiddlewareName
],
rule: rule,
priority: 2000,
...(resource.ssl ? { tls } : {})
@@ -591,7 +704,8 @@ export async function getTraefikConfig(
resource.ssl ? entrypointHttps : entrypointHttp
],
service: maintenanceServiceName,
rule: `${rule} && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`))`,
middlewares: [maintenanceHeadersMiddlewareName],
rule: `${rule} && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`) || Path(\`/favicon.ico\`)) `,
priority: 2001,
...(resource.ssl ? { tls } : {})
};
@@ -845,13 +959,13 @@ export async function getTraefikConfig(
serviceName
].loadBalancer.serversTransport = transportName;
}
} else {
} else if (resource.mode == "tcp" || resource.mode == "udp") {
// Non-HTTP (TCP/UDP) configuration
if (!resource.enableProxy) {
continue;
}
const protocol = resource.protocol.toLowerCase();
const protocol = resource.mode == "udp" ? "udp" : "tcp";
const port = resource.proxyPort;
if (!port) {
@@ -946,6 +1060,293 @@ export async function getTraefikConfig(
}
}
if (browserGatewayUiUrl) {
// Generate Traefik config for browser gateway resources
const browserGatewayPort = 39999;
for (const [, bgResource] of browserGatewayResourcesMap.entries()) {
if (!bgResource.enabled) continue;
if (!bgResource.domainId) continue;
if (!bgResource.fullDomain) continue;
if (!config_output.http.routers) config_output.http.routers = {};
if (!config_output.http.services) config_output.http.services = {};
const fullDomain = bgResource.fullDomain;
const additionalMiddlewares =
config.getRawConfig().traefik.additional_middlewares || [];
const routerMiddlewares = [
badgerMiddlewareName,
...additionalMiddlewares
];
const hostRule = `Host(\`${fullDomain}\`)`;
// Build TLS config
let tls = {};
if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) {
const domainParts = fullDomain.split(".");
let wildCard: string;
if (domainParts.length <= 2) {
wildCard = `*.${domainParts.join(".")}`;
} else {
wildCard = `*.${domainParts.slice(1).join(".")}`;
}
if (!bgResource.subdomain) {
wildCard = fullDomain;
}
const globalDefaultResolver =
config.getRawConfig().traefik.cert_resolver;
const globalDefaultPreferWildcard =
config.getRawConfig().traefik.prefer_wildcard_cert;
const resolverName = bgResource.domainCertResolver
? bgResource.domainCertResolver.trim()
: globalDefaultResolver;
const preferWildcard =
bgResource.preferWildcardCert !== undefined &&
bgResource.preferWildcardCert !== null
? bgResource.preferWildcardCert
: globalDefaultPreferWildcard;
tls = {
certResolver: resolverName,
...(preferWildcard ? { domains: [{ main: wildCard }] } : {})
};
} else {
const matchingCert = validCerts.find(
(cert) => cert.queriedDomain === fullDomain
);
if (!matchingCert) {
logger.debug(
`No matching certificate found for browser gateway domain: ${fullDomain}`
);
continue;
}
}
const bgUiServiceName = `bg-r${bgResource.resourceId}-ui-service`;
if (bgResource.ssl) {
const redirectRouterName = `bg-r${bgResource.resourceId}-redirect`;
config_output.http.routers![redirectRouterName] = {
entryPoints: [
config.getRawConfig().traefik.http_entrypoint
],
middlewares: [redirectHttpsMiddlewareName],
service: bgUiServiceName,
rule: hostRule,
priority: 100
};
}
// Collect online sites for this resource (for any type)
const anySiteOnline = bgResource.targets.some((t) => t.siteOnline);
// Maintenance page logic for browser gateway resources
let showBgMaintenancePage = false;
if (bgResource.maintenanceModeEnabled) {
if (bgResource.maintenanceModeType === "forced") {
showBgMaintenancePage = true;
} else if (bgResource.maintenanceModeType === "automatic") {
showBgMaintenancePage = !anySiteOnline;
}
}
if (showBgMaintenancePage && maintenancePageUiUrl) {
const bgMaintenanceServiceName = `bg-r${bgResource.resourceId}-maintenance-service`;
const bgMaintenanceRouterName = `bg-r${bgResource.resourceId}-maintenance-router`;
const bgRewriteMiddlewareName = `bg-r${bgResource.resourceId}-maintenance-rewrite`;
const bgMaintenanceHeadersMiddlewareName = `bg-r${bgResource.resourceId}-maintenance-headers`;
const entrypointHttp =
config.getRawConfig().traefik.http_entrypoint;
const entrypointHttps =
config.getRawConfig().traefik.https_entrypoint;
if (!config_output.http.services)
config_output.http.services = {};
if (!config_output.http.middlewares)
config_output.http.middlewares = {};
if (!config_output.http.routers)
config_output.http.routers = {};
config_output.http.services![bgMaintenanceServiceName] = {
loadBalancer: {
servers: [
{
url: maintenancePageUiUrl
}
],
passHostHeader: true
}
};
config_output.http.middlewares![bgRewriteMiddlewareName] = {
replacePathRegex: {
regex: "^/(.*)",
replacement: "/maintenance-screen"
}
};
config_output.http.middlewares![
bgMaintenanceHeadersMiddlewareName
] = {
headers: {
customRequestHeaders: {
Host: "app.pangolin.net", // if we are sending to the cloud the host needs to be this but we will pull the p-host to find the resource
"p-host": fullDomain
}
}
};
config_output.http.routers![bgMaintenanceRouterName] = {
entryPoints: [
bgResource.ssl ? entrypointHttps : entrypointHttp
],
service: bgMaintenanceServiceName,
middlewares: [
bgRewriteMiddlewareName,
bgMaintenanceHeadersMiddlewareName
],
rule: hostRule,
priority: 2000,
...(bgResource.ssl ? { tls } : {})
};
config_output.http.routers![
`${bgMaintenanceRouterName}-assets`
] = {
entryPoints: [
bgResource.ssl ? entrypointHttps : entrypointHttp
],
service: bgMaintenanceServiceName,
middlewares: [bgMaintenanceHeadersMiddlewareName],
rule: `${hostRule} && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`) || Path(\`/favicon.ico\`))`,
priority: 2001,
...(bgResource.ssl ? { tls } : {})
};
continue;
}
// Group targets by type and generate per-type websocket routers and services
const typeMap = new Map<string, typeof bgResource.targets>();
for (const t of bgResource.targets) {
if (!typeMap.has(t.bgType)) typeMap.set(t.bgType, []);
typeMap.get(t.bgType)!.push(t);
}
for (const [bgType, typedTargets] of typeMap.entries()) {
const bgKey = `bg-r${bgResource.resourceId}-${bgType}`;
const bgRouterName = `${bgKey}-router`;
const bgServiceName = `${bgKey}-service`;
const bgRule = `${hostRule} && PathPrefix(\`/gateway/${bgType}\`)`;
const servers = typedTargets
.filter((t) => {
if (!t.siteOnline && anySiteOnline) return false;
if (t.siteType === "newt") return !!t.subnet;
return false; // browser gateway only supported on newt sites
})
.map((t) => ({
url: `http://${t.subnet!.split("/")[0]}:${browserGatewayPort}`
}))
.filter(
(v, i, a) => a.findIndex((u) => u.url === v.url) === i
);
config_output.http.routers![bgRouterName] = {
entryPoints: [
bgResource.ssl
? config.getRawConfig().traefik.https_entrypoint
: config.getRawConfig().traefik.http_entrypoint
],
middlewares: routerMiddlewares,
service: bgServiceName,
rule: bgRule,
priority: 110, // highest - websocket path takes precedence
...(bgResource.ssl ? { tls } : {})
};
config_output.http.services![bgServiceName] = {
loadBalancer: {
servers
}
};
}
// UI: serve the browser gateway page from the internal pangolin instance.
// The primary type is used for the path rewrite (e.g. /rdp), mirroring
// how the maintenance page rewrites everything to /maintenance-screen.
const primaryType = typeMap.keys().next().value as string;
const uiRewriteMiddlewareName = `bg-r${bgResource.resourceId}-ui-rewrite`;
const uiHeadersMiddlewareName = `bg-r${bgResource.resourceId}-ui-headers`;
const entrypoint = bgResource.ssl
? config.getRawConfig().traefik.https_entrypoint
: config.getRawConfig().traefik.http_entrypoint;
if (!config_output.http.middlewares) {
config_output.http.middlewares = {};
}
config_output.http.middlewares![uiRewriteMiddlewareName] = {
replacePathRegex: {
regex: "^/(.*)",
replacement: `/${primaryType}`
}
};
config_output.http.middlewares![uiHeadersMiddlewareName] = {
headers: {
customRequestHeaders: {
Host: "app.pangolin.net", // if we are sending to the cloud the host needs to be this but we will pull the p-host to find the resource
"p-host": fullDomain
}
}
};
config_output.http.services![bgUiServiceName] = {
loadBalancer: {
servers: [
{
url: browserGatewayUiUrl
}
]
}
};
// Assets router at higher priority so /_next files load without rewrite.
// Do NOT apply the path-rewrite middleware here — static assets must
// keep their original path; only the host headers are needed.
config_output.http.routers![
`bg-r${bgResource.resourceId}-assets-router`
] = {
entryPoints: [entrypoint],
middlewares: [...routerMiddlewares, uiHeadersMiddlewareName],
service: bgUiServiceName,
rule: `${hostRule} && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`) || Path(\`/favicon.ico\`))`,
priority: 101,
...(bgResource.ssl ? { tls } : {})
};
// Catch-all router rewrites everything on the domain to /{primaryType}
config_output.http.routers![
`bg-r${bgResource.resourceId}-ui-router`
] = {
entryPoints: [entrypoint],
middlewares: [
...routerMiddlewares,
uiRewriteMiddlewareName,
uiHeadersMiddlewareName
],
service: bgUiServiceName,
rule: hostRule,
priority: 100,
...(bgResource.ssl ? { tls } : {})
};
}
}
// Add Traefik routes for siteResource aliases (HTTP mode + SSL) so that
// Traefik generates TLS certificates for those domains even when no
// matching resource exists yet.
@@ -970,10 +1371,6 @@ export async function getTraefikConfig(
const siteResourceRouterName = `${srKey}-router`;
const siteResourceRewriteMiddlewareName = `${srKey}-rewrite`;
const maintenancePort = config.getRawConfig().server.next_port;
const maintenanceHost =
config.getRawConfig().server.internal_hostname;
if (!config_output.http.routers) {
config_output.http.routers = {};
}
@@ -989,7 +1386,7 @@ export async function getTraefikConfig(
loadBalancer: {
servers: [
{
url: `http://${maintenanceHost}:${maintenancePort}`
url: maintenancePageUiUrl
}
],
passHostHeader: true
@@ -1061,7 +1458,7 @@ export async function getTraefikConfig(
config_output.http.routers[`${siteResourceRouterName}-assets`] = {
entryPoints: [config.getRawConfig().traefik.https_entrypoint],
service: siteResourceServiceName,
rule: `Host(\`${fullDomain}\`) && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`))`,
rule: `Host(\`${fullDomain}\`) && (PathPrefix(\`/_next\`) || PathRegexp(\`^/__nextjs*\`) || Path(\`/favicon.ico\`))`,
priority: 101,
tls
};
@@ -1164,7 +1561,7 @@ export async function getTraefikConfig(
config.getRawConfig().traefik.https_entrypoint
],
service: "landing-service",
rule: `Host(\`${fullDomain}\`) && (PathRegexp(\`^/auth/resource/[^/]+$\`) || PathRegexp(\`^/auth/idp/[0-9]+/oidc/callback\`) || PathPrefix(\`/_next\`) || Path(\`/auth/org\`) || PathRegexp(\`^/__nextjs*\`))`,
rule: `Host(\`${fullDomain}\`) && (PathRegexp(\`^/auth/resource/[^/]+$\`) || PathRegexp(\`^/auth/idp/[0-9]+/oidc/callback\`) || PathPrefix(\`/_next\`) || Path(\`/auth/org\`) || PathRegexp(\`^/__nextjs*\`) || Path(\`/favicon.ico\`))`,
priority: 203,
tls: tls
};

Some files were not shown because too many files have changed in this diff Show More