pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-05-08 17:29:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make the rebuild long running function background

Browse Source
This commit is contained in:
Owen
2026-05-07 15:40:34 -07:00
parent 2a9481023a
commit 4eb49e3e60
16 changed files with 230 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/lib/calculateUserClientsForOrgs.ts
View File

@@ -25,9 +25,9 @@ import { tierMatrix } from "./billing/tierMatrix";
export async function calculateUserClientsForOrgs(
userId: string,
trx?: Transaction
trx: Transaction | typeof db = db
): Promise<void> {
const execute = async (transaction: Transaction) => {
const execute = async (transaction: Transaction | typeof db) => {
const orgCache = new Map<string, typeof orgs.$inferSelect | null>();
const adminRoleCache = new Map<
string,
@@ -437,7 +437,7 @@ export async function calculateUserClientsForOrgs(
async function cleanupOrphanedClients(
userId: string,
trx: Transaction,
trx: Transaction | typeof db,
userOrgIds: string[] = []
): Promise<void> {
// Find all OLM clients for this user that should be deleted

24
server/private/routers/user/addUserRole.ts
View File

@@ -14,7 +14,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import stoi from "@server/lib/stoi";
import { clients, db } from "@server/db";
import { clients, db, primaryDb, Client } from "@server/db";
import { userOrgRoles, userOrgs, roles } from "@server/db";
import { eq, and } from "drizzle-orm";
import response from "@server/lib/response";
@@ -122,8 +122,12 @@ export async function addUserRole(
);
}
let newUserRole: { userId: string; orgId: string; roleId: number } | null =
null;
let newUserRole: {
userId: string;
orgId: string;
roleId: number;
} | null = null;
let orgClientsToRebuild: Client[] = [];
await db.transaction(async (trx) => {
const inserted = await trx
.insert(userOrgRoles)
@@ -149,11 +153,19 @@ export async function addUserRole(
)
);
for (const orgClient of orgClients) {
await rebuildClientAssociationsFromClient(orgClient, trx);
}
orgClientsToRebuild = orgClients;
});
for (const orgClient of orgClientsToRebuild) {
rebuildClientAssociationsFromClient(orgClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations for client ${orgClient.clientId} after adding role: ${e}`
);
}
);
}
return response(res, {
data: newUserRole ?? { userId, orgId: role.orgId, roleId },
success: true,

17
server/private/routers/user/removeUserRole.ts
View File

@@ -14,7 +14,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import stoi from "@server/lib/stoi";
import { db } from "@server/db";
import { db, primaryDb, Client } from "@server/db";
import { userOrgRoles, userOrgs, roles, clients } from "@server/db";
import { eq, and } from "drizzle-orm";
import response from "@server/lib/response";
@@ -129,6 +129,7 @@ export async function removeUserRole(
}
}
let orgClientsToRebuild: Client[] = [];
await db.transaction(async (trx) => {
await trx
.delete(userOrgRoles)
@@ -150,11 +151,19 @@ export async function removeUserRole(
)
);
for (const orgClient of orgClients) {
await rebuildClientAssociationsFromClient(orgClient, trx);
}
orgClientsToRebuild = orgClients;
});
for (const orgClient of orgClientsToRebuild) {
rebuildClientAssociationsFromClient(orgClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations for client ${orgClient.clientId} after removing role: ${e}`
);
}
);
}
return response(res, {
data: { userId, orgId: role.orgId, roleId },
success: true,

17
server/private/routers/user/setUserOrgRoles.ts
View File

@@ -13,7 +13,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { clients, db } from "@server/db";
import { clients, db, primaryDb, Client } from "@server/db";
import { userOrgRoles, userOrgs, roles } from "@server/db";
import { eq, and, inArray } from "drizzle-orm";
import response from "@server/lib/response";
@@ -115,6 +115,7 @@ export async function setUserOrgRoles(
);
}
let orgClientsToRebuild: Client[] = [];
await db.transaction(async (trx) => {
await trx
.delete(userOrgRoles)
@@ -142,11 +143,19 @@ export async function setUserOrgRoles(
and(eq(clients.userId, userId), eq(clients.orgId, orgId))
);
for (const orgClient of orgClients) {
await rebuildClientAssociationsFromClient(orgClient, trx);
}
orgClientsToRebuild = orgClients;
});
for (const orgClient of orgClientsToRebuild) {
rebuildClientAssociationsFromClient(orgClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations for client ${orgClient.clientId} after setting roles: ${e}`
);
}
);
}
return response(res, {
data: { userId, orgId, roleIds: uniqueRoleIds },
success: true,

9
server/routers/auth/deleteMyAccount.ts
View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, orgs, userOrgs, users } from "@server/db";
import { db, orgs, userOrgs, users, primaryDb } from "@server/db";
import { eq, and, inArray, not } from "drizzle-orm";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
@@ -218,13 +218,18 @@ export async function deleteMyAccount(
await db.transaction(async (trx) => {
await trx.delete(users).where(eq(users.userId, userId));
await calculateUserClientsForOrgs(userId, trx);
// loop through the other orgs and decrement the count
for (const userOrg of otherOrgsTheUserWasIn) {
await usageService.add(userOrg.orgId, FeatureId.USERS, -1, trx);
}
});
calculateUserClientsForOrgs(userId, primaryDb).catch((e) => {
logger.error(
`Failed to calculate user clients after deleting account for user ${userId}: ${e}`
);
});
try {
await invalidateSession(session.sessionId);
} catch (error) {

24
server/routers/client/createClient.ts
View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { db, primaryDb } from "@server/db";
import {
roles,
Client,
@@ -92,7 +92,10 @@ export async function createClient(
const { orgId } = parsedParams.data;
if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
if (
req.user &&
(!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)
) {
return next(
createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
);
@@ -198,7 +201,10 @@ export async function createClient(
if (!randomExitNode) {
return next(
createHttpError(HttpCode.NOT_FOUND, `No exit nodes available. ${build == "saas" ? "Please contact support." : "You need to install gerbil to use the clients."}`)
createHttpError(
HttpCode.NOT_FOUND,
`No exit nodes available. ${build == "saas" ? "Please contact support." : "You need to install gerbil to use the clients."}`
)
);
}
@@ -256,10 +262,18 @@ export async function createClient(
clientId: newClient.clientId,
dateCreated: moment().toISOString()
});
await rebuildClientAssociationsFromClient(newClient, trx);
});
if (newClient) {
rebuildClientAssociationsFromClient(newClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations after creating client: ${e}`
);
}
);
}
return response<CreateClientResponse>(res, {
data: newClient,
success: true,

14
server/routers/client/createUserClient.ts
View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { db, primaryDb } from "@server/db";
import {
roles,
Client,
@@ -237,10 +237,18 @@ export async function createUserClient(
userId,
clientId: newClient.clientId
});
await rebuildClientAssociationsFromClient(newClient, trx);
});
if (newClient) {
rebuildClientAssociationsFromClient(newClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations after creating user client: ${e}`
);
}
);
}
return response<CreateClientAndOlmResponse>(res, {
data: newClient,
success: true,

36
server/routers/client/deleteClient.ts
View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, olms } from "@server/db";
import { db, olms, primaryDb, Client, Olm } from "@server/db";
import { clients, clientSitesAssociationsCache } from "@server/db";
import { eq } from "drizzle-orm";
import response from "@server/lib/response";
@@ -71,14 +71,17 @@ export async function deleteClient(
);
}
let deletedClient: Client | undefined;
let olm: Olm | undefined;
await db.transaction(async (trx) => {
// Then delete the client itself
const [deletedClient] = await trx
[deletedClient] = await trx
.delete(clients)
.where(eq(clients.clientId, clientId))
.returning();
const [olm] = await trx
[olm] = await trx
.select()
.from(olms)
.where(eq(olms.clientId, clientId))
@@ -88,14 +91,29 @@ export async function deleteClient(
if (!client.userId && client.olmId) {
await trx.delete(olms).where(eq(olms.olmId, client.olmId));
}
await rebuildClientAssociationsFromClient(deletedClient, trx);
if (olm) {
await sendTerminateClient(deletedClient.clientId, OlmErrorCodes.TERMINATED_DELETED, olm.olmId); // the olmId needs to be provided because it cant look it up after deletion
}
});
if (deletedClient) {
rebuildClientAssociationsFromClient(deletedClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations after deleting client ${clientId}: ${e}`
);
}
);
if (olm) {
sendTerminateClient(
deletedClient.clientId,
OlmErrorCodes.TERMINATED_DELETED,
olm.olmId
).catch((e) => {
logger.error(
`Failed to send terminate message for client ${deletedClient?.clientId} after deleting client ${clientId}: ${e}`
);
});
}
}
return response(res, {
data: null,
success: true,

23
server/routers/olm/createUserOlm.ts
View File

@@ -1,5 +1,5 @@
import { NextFunction, Request, Response } from "express";
import { db, olms } from "@server/db";
import { db, olms, primaryDb } from "@server/db";
import HttpCode from "@server/types/HttpCode";
import { z } from "zod";
import createHttpError from "http-errors";
@@ -81,16 +81,19 @@ export async function createUserOlm(
const secretHash = await hashPassword(secret);
await db.transaction(async (trx) => {
await trx.insert(olms).values({
olmId: olmId,
userId,
name,
secretHash,
dateCreated: moment().toISOString()
});
await db.insert(olms).values({
olmId: olmId,
userId,
name,
secretHash,
dateCreated: moment().toISOString()
});
await calculateUserClientsForOrgs(userId, trx);
calculateUserClientsForOrgs(userId, primaryDb).catch((e) => {
console.error(
"Error calculating user clients after creating olm:",
e
);
});
return response<CreateOlmResponse>(res, {

39
server/routers/olm/deleteUserOlm.ts
View File

@@ -1,5 +1,5 @@
import { NextFunction, Request, Response } from "express";
import { Client, db } from "@server/db";
import { Client, db, Olm, primaryDb } from "@server/db";
import { olms, clients, clientSitesAssociationsCache } from "@server/db";
import { eq } from "drizzle-orm";
import HttpCode from "@server/types/HttpCode";
@@ -49,6 +49,7 @@ export async function deleteUserOlm(
const { olmId } = parsedParams.data;
let deletedClient: Client | undefined;
// Delete associated clients and the OLM in a transaction
await db.transaction(async (trx) => {
// Find all clients associated with this OLM
@@ -57,7 +58,6 @@ export async function deleteUserOlm(
.from(clients)
.where(eq(clients.olmId, olmId));
let deletedClient: Client | null = null;
// Delete all associated clients
if (associatedClients.length > 0) {
[deletedClient] = await trx
@@ -67,23 +67,28 @@ export async function deleteUserOlm(
}
// Finally, delete the OLM itself
const [olm] = await trx
.delete(olms)
.where(eq(olms.olmId, olmId))
.returning();
if (deletedClient) {
await rebuildClientAssociationsFromClient(deletedClient, trx);
if (olm) {
await sendTerminateClient(
deletedClient.clientId,
OlmErrorCodes.TERMINATED_DELETED,
olm.olmId
); // the olmId needs to be provided because it cant look it up after deletion
}
}
await trx.delete(olms).where(eq(olms.olmId, olmId)).returning();
});
if (deletedClient) {
rebuildClientAssociationsFromClient(deletedClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client-site associations after deleting OLM ${olmId}: ${e}`
);
}
);
sendTerminateClient(
deletedClient.clientId,
OlmErrorCodes.TERMINATED_DELETED,
olmId
).catch((e) => {
logger.error(
`Failed to send terminate message for client ${deletedClient?.clientId} after deleting OLM ${olmId}: ${e}`
);
});
}
return response(res, {
data: null,
success: true,

9
server/routers/siteResource/batchAddClientToSiteResources.ts
View File

@@ -5,7 +5,8 @@ import {
clients,
clientSiteResources,
siteResources,
apiKeyOrg
apiKeyOrg,
primaryDb
} from "@server/db";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
@@ -220,8 +221,12 @@ export async function batchAddClientToSiteResources(
siteResourceId: siteResource.siteResourceId
});
}
});
await rebuildClientAssociationsFromClient(client, trx);
rebuildClientAssociationsFromClient(client, primaryDb).catch((e) => {
logger.error(
`Failed to rebuild client associations after batch adding site resources for client ${clientId}: ${e}`
);
});
return response(res, {

24
server/routers/user/acceptInvite.ts
View File

@@ -1,7 +1,13 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, orgs } from "@server/db";
import { roles, userInviteRoles, userInvites, userOrgs, users } from "@server/db";
import { db, orgs, primaryDb } from "@server/db";
import {
roles,
userInviteRoles,
userInvites,
userOrgs,
users
} from "@server/db";
import { eq, and, inArray } from "drizzle-orm";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
@@ -146,9 +152,7 @@ export async function acceptInvite(
.from(userInviteRoles)
.where(eq(userInviteRoles.inviteId, inviteId));
const inviteRoleIds = [
...new Set(inviteRoleRows.map((r) => r.roleId))
];
const inviteRoleIds = [...new Set(inviteRoleRows.map((r) => r.roleId))];
if (inviteRoleIds.length === 0) {
return next(
createHttpError(
@@ -193,13 +197,19 @@ export async function acceptInvite(
.delete(userInvites)
.where(eq(userInvites.inviteId, inviteId));
await calculateUserClientsForOrgs(existingUser[0].userId, trx);
logger.debug(
`User ${existingUser[0].userId} accepted invite to org ${existingInvite.orgId}`
);
});
calculateUserClientsForOrgs(existingUser[0].userId, primaryDb).catch(
(e) => {
logger.error(
`Failed to calculate user clients after accepting invite for user ${existingUser[0].userId}: ${e}`
);
}
);
return response<AcceptInviteResponse>(res, {
data: { accepted: true, orgId: existingInvite.orgId },
success: true,

18
server/routers/user/addUserRoleLegacy.ts
View File

@@ -1,7 +1,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import stoi from "@server/lib/stoi";
import { clients, db } from "@server/db";
import { clients, db, primaryDb, Client } from "@server/db";
import { userOrgRoles, userOrgs, roles } from "@server/db";
import { eq, and } from "drizzle-orm";
import response from "@server/lib/response";
@@ -112,6 +112,8 @@ export async function addUserRoleLegacy(
);
}
let orgClientsToRebuild: Client[] = [];
await db.transaction(async (trx) => {
await trx
.delete(userOrgRoles)
@@ -138,11 +140,19 @@ export async function addUserRoleLegacy(
)
);
for (const orgClient of orgClients) {
await rebuildClientAssociationsFromClient(orgClient, trx);
}
orgClientsToRebuild = orgClients;
});
for (const orgClient of orgClientsToRebuild) {
rebuildClientAssociationsFromClient(orgClient, primaryDb).catch(
(e) => {
logger.error(
`Failed to rebuild client associations for client ${orgClient.clientId} after adding role: ${e}`
);
}
);
}
return response(res, {
data: { ...existingUser, roleId },
success: true,

8
server/routers/user/adminRemoveUser.ts
View File

@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { db, primaryDb } from "@server/db";
import { users } from "@server/db";
import { eq } from "drizzle-orm";
import response from "@server/lib/response";
@@ -53,8 +53,12 @@ export async function adminRemoveUser(
await db.transaction(async (trx) => {
await trx.delete(users).where(eq(users.userId, userId));
});
await calculateUserClientsForOrgs(userId, trx);
calculateUserClientsForOrgs(userId, primaryDb).catch((e) => {
logger.error(
`Failed to calculate user clients after removing user ${userId}: ${e}`
);
});
return response(res, {

50
server/routers/user/createOrgUser.ts
View File

@@ -6,7 +6,7 @@ import createHttpError from "http-errors";
import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { db, orgs } from "@server/db";
import { db, orgs, primaryDb } from "@server/db";
import { and, eq, inArray } from "drizzle-orm";
import { idp, idpOidcConfig, roles, userOrgs, users } from "@server/db";
import { generateId } from "@server/auth/sessions/app";
@@ -34,8 +34,7 @@ const bodySchema = z
roleId: z.number().int().positive().optional()
})
.refine(
(d) =>
(d.roleIds != null && d.roleIds.length > 0) || d.roleId != null,
(d) => (d.roleIds != null && d.roleIds.length > 0) || d.roleId != null,
{ message: "roleIds or roleId is required", path: ["roleIds"] }
)
.transform((data) => ({
@@ -100,8 +99,14 @@ export async function createOrgUser(
}
const { orgId } = parsedParams.data;
const { username, email, name, type, idpId, roleIds: uniqueRoleIds } =
parsedBody.data;
const {
username,
email,
name,
type,
idpId,
roleIds: uniqueRoleIds
} = parsedBody.data;
if (build == "saas") {
const usage = await usageService.getUsage(orgId, FeatureId.USERS);
@@ -232,6 +237,7 @@ export async function createOrgUser(
);
}
let userIdForClients: string | undefined;
await db.transaction(async (trx) => {
const [existingUser] = await trx
.select()
@@ -270,7 +276,7 @@ export async function createOrgUser(
{
orgId,
userId: existingUser.userId,
autoProvisioned: false,
autoProvisioned: false
},
uniqueRoleIds,
trx
@@ -292,20 +298,30 @@ export async function createOrgUser(
})
.returning();
await assignUserToOrg(
org,
{
orgId,
userId: newUser.userId,
autoProvisioned: false,
},
uniqueRoleIds,
trx
);
await assignUserToOrg(
org,
{
orgId,
userId: newUser.userId,
autoProvisioned: false
},
uniqueRoleIds,
trx
);
}
await calculateUserClientsForOrgs(userId, trx);
userIdForClients = userId;
});
if (userIdForClients) {
calculateUserClientsForOrgs(userIdForClients, primaryDb).catch(
(e) => {
logger.error(
`Failed to calculate user clients after creating org user: ${e}`
);
}
);
}
} else {
return next(
createHttpError(HttpCode.BAD_REQUEST, "User type is required")

26
server/routers/user/removeUserOrg.ts
View File

@@ -7,7 +7,8 @@ import {
siteResources,
sites,
UserOrg,
userSiteResources
userSiteResources,
primaryDb
} from "@server/db";
import { userOrgs, userResources, users, userSites } from "@server/db";
import { and, count, eq, exists, inArray } from "drizzle-orm";
@@ -91,25 +92,12 @@ export async function removeUserOrg(
await db.transaction(async (trx) => {
await removeUserFromOrg(org, userId, trx);
});
// if (build === "saas") {
// const [rootUser] = await trx
// .select()
// .from(users)
// .where(eq(users.userId, userId));
//
// const [leftInOrgs] = await trx
// .select({ count: count() })
// .from(userOrgs)
// .where(eq(userOrgs.userId, userId));
//
// // if the user is not an internal user and does not belong to any org, delete the entire user
// if (rootUser?.type !== UserType.Internal && !leftInOrgs.count) {
// await trx.delete(users).where(eq(users.userId, userId));
// }
// }
await calculateUserClientsForOrgs(userId, trx);
calculateUserClientsForOrgs(userId, primaryDb).catch((e) => {
logger.error(
`Failed to calculate user clients after removing user ${userId} from org ${orgId}: ${e}`
);
});
return response(res, {