mirror of
https://github.com/fosrl/pangolin.git
synced 2026-03-30 22:46:40 +00:00
Merge branch 'logging-provision' into dev
This commit is contained in:
@@ -14,12 +14,14 @@
|
||||
import { rateLimitService } from "#private/lib/rateLimit";
|
||||
import { cleanup as wsCleanup } from "#private/routers/ws";
|
||||
import { flushBandwidthToDb } from "@server/routers/newt/handleReceiveBandwidthMessage";
|
||||
import { flushConnectionLogToDb } from "#dynamic/routers/newt";
|
||||
import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth";
|
||||
import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator";
|
||||
|
||||
async function cleanup() {
|
||||
await stopPingAccumulator();
|
||||
await flushBandwidthToDb();
|
||||
await flushConnectionLogToDb();
|
||||
await flushSiteBandwidthToDb();
|
||||
await rateLimitService.cleanup();
|
||||
await wsCleanup();
|
||||
@@ -31,4 +33,4 @@ export async function initCleanup() {
|
||||
// Handle process termination
|
||||
process.on("SIGTERM", () => cleanup());
|
||||
process.on("SIGINT", () => cleanup());
|
||||
}
|
||||
}
|
||||
|
||||
99
server/private/routers/auditLogs/exportConnectionAuditLog.ts
Normal file
99
server/private/routers/auditLogs/exportConnectionAuditLog.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
import { registry } from "@server/openApi";
|
||||
import { NextFunction } from "express";
|
||||
import { Request, Response } from "express";
|
||||
import { OpenAPITags } from "@server/openApi";
|
||||
import createHttpError from "http-errors";
|
||||
import HttpCode from "@server/types/HttpCode";
|
||||
import { fromError } from "zod-validation-error";
|
||||
import logger from "@server/logger";
|
||||
import {
|
||||
queryConnectionAuditLogsParams,
|
||||
queryConnectionAuditLogsQuery,
|
||||
queryConnection,
|
||||
countConnectionQuery
|
||||
} from "./queryConnectionAuditLog";
|
||||
import { generateCSV } from "@server/routers/auditLogs/generateCSV";
|
||||
import { MAX_EXPORT_LIMIT } from "@server/routers/auditLogs";
|
||||
|
||||
registry.registerPath({
|
||||
method: "get",
|
||||
path: "/org/{orgId}/logs/connection/export",
|
||||
description: "Export the connection audit log for an organization as CSV",
|
||||
tags: [OpenAPITags.Logs],
|
||||
request: {
|
||||
query: queryConnectionAuditLogsQuery,
|
||||
params: queryConnectionAuditLogsParams
|
||||
},
|
||||
responses: {}
|
||||
});
|
||||
|
||||
export async function exportConnectionAuditLogs(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<any> {
|
||||
try {
|
||||
const parsedQuery = queryConnectionAuditLogsQuery.safeParse(req.query);
|
||||
if (!parsedQuery.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedQuery.error)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const parsedParams = queryConnectionAuditLogsParams.safeParse(req.params);
|
||||
if (!parsedParams.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedParams.error)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const data = { ...parsedQuery.data, ...parsedParams.data };
|
||||
const [{ count }] = await countConnectionQuery(data);
|
||||
if (count > MAX_EXPORT_LIMIT) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
`Export limit exceeded. Your selection contains ${count} rows, but the maximum is ${MAX_EXPORT_LIMIT} rows. Please select a shorter time range to reduce the data.`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const baseQuery = queryConnection(data);
|
||||
|
||||
const log = await baseQuery.limit(data.limit).offset(data.offset);
|
||||
|
||||
const csvData = generateCSV(log);
|
||||
|
||||
res.setHeader("Content-Type", "text/csv");
|
||||
res.setHeader(
|
||||
"Content-Disposition",
|
||||
`attachment; filename="connection-audit-logs-${data.orgId}-${Date.now()}.csv"`
|
||||
);
|
||||
|
||||
return res.send(csvData);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return next(
|
||||
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -15,3 +15,5 @@ export * from "./queryActionAuditLog";
|
||||
export * from "./exportActionAuditLog";
|
||||
export * from "./queryAccessAuditLog";
|
||||
export * from "./exportAccessAuditLog";
|
||||
export * from "./queryConnectionAuditLog";
|
||||
export * from "./exportConnectionAuditLog";
|
||||
|
||||
524
server/private/routers/auditLogs/queryConnectionAuditLog.ts
Normal file
524
server/private/routers/auditLogs/queryConnectionAuditLog.ts
Normal file
@@ -0,0 +1,524 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
import {
|
||||
connectionAuditLog,
|
||||
logsDb,
|
||||
siteResources,
|
||||
sites,
|
||||
clients,
|
||||
users,
|
||||
primaryDb
|
||||
} from "@server/db";
|
||||
import { registry } from "@server/openApi";
|
||||
import { NextFunction } from "express";
|
||||
import { Request, Response } from "express";
|
||||
import { eq, gt, lt, and, count, desc, inArray } from "drizzle-orm";
|
||||
import { OpenAPITags } from "@server/openApi";
|
||||
import { z } from "zod";
|
||||
import createHttpError from "http-errors";
|
||||
import HttpCode from "@server/types/HttpCode";
|
||||
import { fromError } from "zod-validation-error";
|
||||
import { QueryConnectionAuditLogResponse } from "@server/routers/auditLogs/types";
|
||||
import response from "@server/lib/response";
|
||||
import logger from "@server/logger";
|
||||
import { getSevenDaysAgo } from "@app/lib/getSevenDaysAgo";
|
||||
|
||||
export const queryConnectionAuditLogsQuery = z.object({
|
||||
// iso string just validate its a parseable date
|
||||
timeStart: z
|
||||
.string()
|
||||
.refine((val) => !isNaN(Date.parse(val)), {
|
||||
error: "timeStart must be a valid ISO date string"
|
||||
})
|
||||
.transform((val) => Math.floor(new Date(val).getTime() / 1000))
|
||||
.prefault(() => getSevenDaysAgo().toISOString())
|
||||
.openapi({
|
||||
type: "string",
|
||||
format: "date-time",
|
||||
description:
|
||||
"Start time as ISO date string (defaults to 7 days ago)"
|
||||
}),
|
||||
timeEnd: z
|
||||
.string()
|
||||
.refine((val) => !isNaN(Date.parse(val)), {
|
||||
error: "timeEnd must be a valid ISO date string"
|
||||
})
|
||||
.transform((val) => Math.floor(new Date(val).getTime() / 1000))
|
||||
.optional()
|
||||
.prefault(() => new Date().toISOString())
|
||||
.openapi({
|
||||
type: "string",
|
||||
format: "date-time",
|
||||
description:
|
||||
"End time as ISO date string (defaults to current time)"
|
||||
}),
|
||||
protocol: z.string().optional(),
|
||||
sourceAddr: z.string().optional(),
|
||||
destAddr: z.string().optional(),
|
||||
clientId: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform(Number)
|
||||
.pipe(z.int().positive())
|
||||
.optional(),
|
||||
siteId: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform(Number)
|
||||
.pipe(z.int().positive())
|
||||
.optional(),
|
||||
siteResourceId: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform(Number)
|
||||
.pipe(z.int().positive())
|
||||
.optional(),
|
||||
userId: z.string().optional(),
|
||||
limit: z
|
||||
.string()
|
||||
.optional()
|
||||
.default("1000")
|
||||
.transform(Number)
|
||||
.pipe(z.int().positive()),
|
||||
offset: z
|
||||
.string()
|
||||
.optional()
|
||||
.default("0")
|
||||
.transform(Number)
|
||||
.pipe(z.int().nonnegative())
|
||||
});
|
||||
|
||||
export const queryConnectionAuditLogsParams = z.object({
|
||||
orgId: z.string()
|
||||
});
|
||||
|
||||
export const queryConnectionAuditLogsCombined =
|
||||
queryConnectionAuditLogsQuery.merge(queryConnectionAuditLogsParams);
|
||||
type Q = z.infer<typeof queryConnectionAuditLogsCombined>;
|
||||
|
||||
function getWhere(data: Q) {
|
||||
return and(
|
||||
gt(connectionAuditLog.startedAt, data.timeStart),
|
||||
lt(connectionAuditLog.startedAt, data.timeEnd),
|
||||
eq(connectionAuditLog.orgId, data.orgId),
|
||||
data.protocol
|
||||
? eq(connectionAuditLog.protocol, data.protocol)
|
||||
: undefined,
|
||||
data.sourceAddr
|
||||
? eq(connectionAuditLog.sourceAddr, data.sourceAddr)
|
||||
: undefined,
|
||||
data.destAddr
|
||||
? eq(connectionAuditLog.destAddr, data.destAddr)
|
||||
: undefined,
|
||||
data.clientId
|
||||
? eq(connectionAuditLog.clientId, data.clientId)
|
||||
: undefined,
|
||||
data.siteId
|
||||
? eq(connectionAuditLog.siteId, data.siteId)
|
||||
: undefined,
|
||||
data.siteResourceId
|
||||
? eq(connectionAuditLog.siteResourceId, data.siteResourceId)
|
||||
: undefined,
|
||||
data.userId
|
||||
? eq(connectionAuditLog.userId, data.userId)
|
||||
: undefined
|
||||
);
|
||||
}
|
||||
|
||||
export function queryConnection(data: Q) {
|
||||
return logsDb
|
||||
.select({
|
||||
sessionId: connectionAuditLog.sessionId,
|
||||
siteResourceId: connectionAuditLog.siteResourceId,
|
||||
orgId: connectionAuditLog.orgId,
|
||||
siteId: connectionAuditLog.siteId,
|
||||
clientId: connectionAuditLog.clientId,
|
||||
userId: connectionAuditLog.userId,
|
||||
sourceAddr: connectionAuditLog.sourceAddr,
|
||||
destAddr: connectionAuditLog.destAddr,
|
||||
protocol: connectionAuditLog.protocol,
|
||||
startedAt: connectionAuditLog.startedAt,
|
||||
endedAt: connectionAuditLog.endedAt,
|
||||
bytesTx: connectionAuditLog.bytesTx,
|
||||
bytesRx: connectionAuditLog.bytesRx
|
||||
})
|
||||
.from(connectionAuditLog)
|
||||
.where(getWhere(data))
|
||||
.orderBy(
|
||||
desc(connectionAuditLog.startedAt),
|
||||
desc(connectionAuditLog.id)
|
||||
);
|
||||
}
|
||||
|
||||
export function countConnectionQuery(data: Q) {
|
||||
const countQuery = logsDb
|
||||
.select({ count: count() })
|
||||
.from(connectionAuditLog)
|
||||
.where(getWhere(data));
|
||||
return countQuery;
|
||||
}
|
||||
|
||||
async function enrichWithDetails(
|
||||
logs: Awaited<ReturnType<typeof queryConnection>>
|
||||
) {
|
||||
// Collect unique IDs from logs
|
||||
const siteResourceIds = [
|
||||
...new Set(
|
||||
logs
|
||||
.map((log) => log.siteResourceId)
|
||||
.filter((id): id is number => id !== null && id !== undefined)
|
||||
)
|
||||
];
|
||||
const siteIds = [
|
||||
...new Set(
|
||||
logs
|
||||
.map((log) => log.siteId)
|
||||
.filter((id): id is number => id !== null && id !== undefined)
|
||||
)
|
||||
];
|
||||
const clientIds = [
|
||||
...new Set(
|
||||
logs
|
||||
.map((log) => log.clientId)
|
||||
.filter((id): id is number => id !== null && id !== undefined)
|
||||
)
|
||||
];
|
||||
const userIds = [
|
||||
...new Set(
|
||||
logs
|
||||
.map((log) => log.userId)
|
||||
.filter((id): id is string => id !== null && id !== undefined)
|
||||
)
|
||||
];
|
||||
|
||||
// Fetch resource details from main database
|
||||
const resourceMap = new Map<
|
||||
number,
|
||||
{ name: string; niceId: string }
|
||||
>();
|
||||
if (siteResourceIds.length > 0) {
|
||||
const resourceDetails = await primaryDb
|
||||
.select({
|
||||
siteResourceId: siteResources.siteResourceId,
|
||||
name: siteResources.name,
|
||||
niceId: siteResources.niceId
|
||||
})
|
||||
.from(siteResources)
|
||||
.where(inArray(siteResources.siteResourceId, siteResourceIds));
|
||||
|
||||
for (const r of resourceDetails) {
|
||||
resourceMap.set(r.siteResourceId, {
|
||||
name: r.name,
|
||||
niceId: r.niceId
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch site details from main database
|
||||
const siteMap = new Map<number, { name: string; niceId: string }>();
|
||||
if (siteIds.length > 0) {
|
||||
const siteDetails = await primaryDb
|
||||
.select({
|
||||
siteId: sites.siteId,
|
||||
name: sites.name,
|
||||
niceId: sites.niceId
|
||||
})
|
||||
.from(sites)
|
||||
.where(inArray(sites.siteId, siteIds));
|
||||
|
||||
for (const s of siteDetails) {
|
||||
siteMap.set(s.siteId, { name: s.name, niceId: s.niceId });
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch client details from main database
|
||||
const clientMap = new Map<
|
||||
number,
|
||||
{ name: string; niceId: string; type: string }
|
||||
>();
|
||||
if (clientIds.length > 0) {
|
||||
const clientDetails = await primaryDb
|
||||
.select({
|
||||
clientId: clients.clientId,
|
||||
name: clients.name,
|
||||
niceId: clients.niceId,
|
||||
type: clients.type
|
||||
})
|
||||
.from(clients)
|
||||
.where(inArray(clients.clientId, clientIds));
|
||||
|
||||
for (const c of clientDetails) {
|
||||
clientMap.set(c.clientId, {
|
||||
name: c.name,
|
||||
niceId: c.niceId,
|
||||
type: c.type
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch user details from main database
|
||||
const userMap = new Map<
|
||||
string,
|
||||
{ email: string | null }
|
||||
>();
|
||||
if (userIds.length > 0) {
|
||||
const userDetails = await primaryDb
|
||||
.select({
|
||||
userId: users.userId,
|
||||
email: users.email
|
||||
})
|
||||
.from(users)
|
||||
.where(inArray(users.userId, userIds));
|
||||
|
||||
for (const u of userDetails) {
|
||||
userMap.set(u.userId, { email: u.email });
|
||||
}
|
||||
}
|
||||
|
||||
// Enrich logs with details
|
||||
return logs.map((log) => ({
|
||||
...log,
|
||||
resourceName: log.siteResourceId
|
||||
? resourceMap.get(log.siteResourceId)?.name ?? null
|
||||
: null,
|
||||
resourceNiceId: log.siteResourceId
|
||||
? resourceMap.get(log.siteResourceId)?.niceId ?? null
|
||||
: null,
|
||||
siteName: log.siteId
|
||||
? siteMap.get(log.siteId)?.name ?? null
|
||||
: null,
|
||||
siteNiceId: log.siteId
|
||||
? siteMap.get(log.siteId)?.niceId ?? null
|
||||
: null,
|
||||
clientName: log.clientId
|
||||
? clientMap.get(log.clientId)?.name ?? null
|
||||
: null,
|
||||
clientNiceId: log.clientId
|
||||
? clientMap.get(log.clientId)?.niceId ?? null
|
||||
: null,
|
||||
clientType: log.clientId
|
||||
? clientMap.get(log.clientId)?.type ?? null
|
||||
: null,
|
||||
userEmail: log.userId
|
||||
? userMap.get(log.userId)?.email ?? null
|
||||
: null
|
||||
}));
|
||||
}
|
||||
|
||||
async function queryUniqueFilterAttributes(
|
||||
timeStart: number,
|
||||
timeEnd: number,
|
||||
orgId: string
|
||||
) {
|
||||
const baseConditions = and(
|
||||
gt(connectionAuditLog.startedAt, timeStart),
|
||||
lt(connectionAuditLog.startedAt, timeEnd),
|
||||
eq(connectionAuditLog.orgId, orgId)
|
||||
);
|
||||
|
||||
// Get unique protocols
|
||||
const uniqueProtocols = await logsDb
|
||||
.selectDistinct({
|
||||
protocol: connectionAuditLog.protocol
|
||||
})
|
||||
.from(connectionAuditLog)
|
||||
.where(baseConditions);
|
||||
|
||||
// Get unique destination addresses
|
||||
const uniqueDestAddrs = await logsDb
|
||||
.selectDistinct({
|
||||
destAddr: connectionAuditLog.destAddr
|
||||
})
|
||||
.from(connectionAuditLog)
|
||||
.where(baseConditions);
|
||||
|
||||
// Get unique client IDs
|
||||
const uniqueClients = await logsDb
|
||||
.selectDistinct({
|
||||
clientId: connectionAuditLog.clientId
|
||||
})
|
||||
.from(connectionAuditLog)
|
||||
.where(baseConditions);
|
||||
|
||||
// Get unique resource IDs
|
||||
const uniqueResources = await logsDb
|
||||
.selectDistinct({
|
||||
siteResourceId: connectionAuditLog.siteResourceId
|
||||
})
|
||||
.from(connectionAuditLog)
|
||||
.where(baseConditions);
|
||||
|
||||
// Get unique user IDs
|
||||
const uniqueUsers = await logsDb
|
||||
.selectDistinct({
|
||||
userId: connectionAuditLog.userId
|
||||
})
|
||||
.from(connectionAuditLog)
|
||||
.where(baseConditions);
|
||||
|
||||
// Enrich client IDs with names from main database
|
||||
const clientIds = uniqueClients
|
||||
.map((row) => row.clientId)
|
||||
.filter((id): id is number => id !== null);
|
||||
|
||||
let clientsWithNames: Array<{ id: number; name: string }> = [];
|
||||
if (clientIds.length > 0) {
|
||||
const clientDetails = await primaryDb
|
||||
.select({
|
||||
clientId: clients.clientId,
|
||||
name: clients.name
|
||||
})
|
||||
.from(clients)
|
||||
.where(inArray(clients.clientId, clientIds));
|
||||
|
||||
clientsWithNames = clientDetails.map((c) => ({
|
||||
id: c.clientId,
|
||||
name: c.name
|
||||
}));
|
||||
}
|
||||
|
||||
// Enrich resource IDs with names from main database
|
||||
const resourceIds = uniqueResources
|
||||
.map((row) => row.siteResourceId)
|
||||
.filter((id): id is number => id !== null);
|
||||
|
||||
let resourcesWithNames: Array<{ id: number; name: string | null }> = [];
|
||||
if (resourceIds.length > 0) {
|
||||
const resourceDetails = await primaryDb
|
||||
.select({
|
||||
siteResourceId: siteResources.siteResourceId,
|
||||
name: siteResources.name
|
||||
})
|
||||
.from(siteResources)
|
||||
.where(inArray(siteResources.siteResourceId, resourceIds));
|
||||
|
||||
resourcesWithNames = resourceDetails.map((r) => ({
|
||||
id: r.siteResourceId,
|
||||
name: r.name
|
||||
}));
|
||||
}
|
||||
|
||||
// Enrich user IDs with emails from main database
|
||||
const userIdsList = uniqueUsers
|
||||
.map((row) => row.userId)
|
||||
.filter((id): id is string => id !== null);
|
||||
|
||||
let usersWithEmails: Array<{ id: string; email: string | null }> = [];
|
||||
if (userIdsList.length > 0) {
|
||||
const userDetails = await primaryDb
|
||||
.select({
|
||||
userId: users.userId,
|
||||
email: users.email
|
||||
})
|
||||
.from(users)
|
||||
.where(inArray(users.userId, userIdsList));
|
||||
|
||||
usersWithEmails = userDetails.map((u) => ({
|
||||
id: u.userId,
|
||||
email: u.email
|
||||
}));
|
||||
}
|
||||
|
||||
return {
|
||||
protocols: uniqueProtocols
|
||||
.map((row) => row.protocol)
|
||||
.filter((protocol): protocol is string => protocol !== null),
|
||||
destAddrs: uniqueDestAddrs
|
||||
.map((row) => row.destAddr)
|
||||
.filter((addr): addr is string => addr !== null),
|
||||
clients: clientsWithNames,
|
||||
resources: resourcesWithNames,
|
||||
users: usersWithEmails
|
||||
};
|
||||
}
|
||||
|
||||
registry.registerPath({
|
||||
method: "get",
|
||||
path: "/org/{orgId}/logs/connection",
|
||||
description: "Query the connection audit log for an organization",
|
||||
tags: [OpenAPITags.Logs],
|
||||
request: {
|
||||
query: queryConnectionAuditLogsQuery,
|
||||
params: queryConnectionAuditLogsParams
|
||||
},
|
||||
responses: {}
|
||||
});
|
||||
|
||||
export async function queryConnectionAuditLogs(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<any> {
|
||||
try {
|
||||
const parsedQuery = queryConnectionAuditLogsQuery.safeParse(req.query);
|
||||
if (!parsedQuery.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedQuery.error)
|
||||
)
|
||||
);
|
||||
}
|
||||
const parsedParams = queryConnectionAuditLogsParams.safeParse(
|
||||
req.params
|
||||
);
|
||||
if (!parsedParams.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedParams.error)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const data = { ...parsedQuery.data, ...parsedParams.data };
|
||||
|
||||
const baseQuery = queryConnection(data);
|
||||
|
||||
const logsRaw = await baseQuery.limit(data.limit).offset(data.offset);
|
||||
|
||||
// Enrich with resource, site, client, and user details
|
||||
const log = await enrichWithDetails(logsRaw);
|
||||
|
||||
const totalCountResult = await countConnectionQuery(data);
|
||||
const totalCount = totalCountResult[0].count;
|
||||
|
||||
const filterAttributes = await queryUniqueFilterAttributes(
|
||||
data.timeStart,
|
||||
data.timeEnd,
|
||||
data.orgId
|
||||
);
|
||||
|
||||
return response<QueryConnectionAuditLogResponse>(res, {
|
||||
data: {
|
||||
log: log,
|
||||
pagination: {
|
||||
total: totalCount,
|
||||
limit: data.limit,
|
||||
offset: data.offset
|
||||
},
|
||||
filterAttributes
|
||||
},
|
||||
success: true,
|
||||
error: false,
|
||||
message: "Connection audit logs retrieved successfully",
|
||||
status: HttpCode.OK
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return next(
|
||||
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,9 @@ import {
|
||||
resources,
|
||||
roles,
|
||||
siteResources,
|
||||
userOrgRoles
|
||||
userOrgRoles,
|
||||
siteProvisioningKeyOrg,
|
||||
siteProvisioningKeys,
|
||||
} from "@server/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
|
||||
@@ -296,6 +298,10 @@ async function disableFeature(
|
||||
await disableFullRbac(orgId);
|
||||
break;
|
||||
|
||||
case TierFeature.SiteProvisioningKeys:
|
||||
await disableSiteProvisioningKeys(orgId);
|
||||
break;
|
||||
|
||||
default:
|
||||
logger.warn(
|
||||
`Unknown feature ${feature} for org ${orgId}, skipping`
|
||||
@@ -335,6 +341,57 @@ async function disableFullRbac(orgId: string): Promise<void> {
|
||||
logger.info(`Disabled full RBAC for org ${orgId}`);
|
||||
}
|
||||
|
||||
async function disableSiteProvisioningKeys(orgId: string): Promise<void> {
|
||||
const rows = await db
|
||||
.select({
|
||||
siteProvisioningKeyId:
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId
|
||||
})
|
||||
.from(siteProvisioningKeyOrg)
|
||||
.where(eq(siteProvisioningKeyOrg.orgId, orgId));
|
||||
|
||||
for (const { siteProvisioningKeyId } of rows) {
|
||||
await db.transaction(async (trx) => {
|
||||
await trx
|
||||
.delete(siteProvisioningKeyOrg)
|
||||
.where(
|
||||
and(
|
||||
eq(
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
),
|
||||
eq(siteProvisioningKeyOrg.orgId, orgId)
|
||||
)
|
||||
);
|
||||
|
||||
const remaining = await trx
|
||||
.select()
|
||||
.from(siteProvisioningKeyOrg)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
);
|
||||
|
||||
if (remaining.length === 0) {
|
||||
await trx
|
||||
.delete(siteProvisioningKeys)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Removed site provisioning keys for org ${orgId} after tier downgrade`
|
||||
);
|
||||
}
|
||||
|
||||
async function disableLoginPageBranding(orgId: string): Promise<void> {
|
||||
const [existingBranding] = await db
|
||||
.select()
|
||||
|
||||
@@ -27,6 +27,7 @@ import * as reKey from "#private/routers/re-key";
|
||||
import * as approval from "#private/routers/approvals";
|
||||
import * as ssh from "#private/routers/ssh";
|
||||
import * as user from "#private/routers/user";
|
||||
import * as siteProvisioning from "#private/routers/siteProvisioning";
|
||||
|
||||
import {
|
||||
verifyOrgAccess,
|
||||
@@ -37,7 +38,8 @@ import {
|
||||
verifyLimits,
|
||||
verifyRoleAccess,
|
||||
verifyUserAccess,
|
||||
verifyUserCanSetUserOrgRoles
|
||||
verifyUserCanSetUserOrgRoles,
|
||||
verifySiteProvisioningKeyAccess
|
||||
} from "@server/middlewares";
|
||||
import { ActionsEnum } from "@server/auth/actions";
|
||||
import {
|
||||
@@ -482,6 +484,25 @@ authenticated.get(
|
||||
logs.exportAccessAuditLogs
|
||||
);
|
||||
|
||||
authenticated.get(
|
||||
"/org/:orgId/logs/connection",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.connectionLogs),
|
||||
verifyOrgAccess,
|
||||
verifyUserHasAction(ActionsEnum.exportLogs),
|
||||
logs.queryConnectionAuditLogs
|
||||
);
|
||||
|
||||
authenticated.get(
|
||||
"/org/:orgId/logs/connection/export",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.logExport),
|
||||
verifyOrgAccess,
|
||||
verifyUserHasAction(ActionsEnum.exportLogs),
|
||||
logActionAudit(ActionsEnum.exportLogs),
|
||||
logs.exportConnectionAuditLogs
|
||||
);
|
||||
|
||||
authenticated.post(
|
||||
"/re-key/:clientId/regenerate-client-secret",
|
||||
verifyClientAccess, // this is first to set the org id
|
||||
@@ -552,3 +573,45 @@ authenticated.post(
|
||||
logActionAudit(ActionsEnum.setUserOrgRoles),
|
||||
user.setUserOrgRoles
|
||||
);
|
||||
|
||||
authenticated.put(
|
||||
"/org/:orgId/site-provisioning-key",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.siteProvisioningKeys),
|
||||
verifyOrgAccess,
|
||||
verifyLimits,
|
||||
verifyUserHasAction(ActionsEnum.createSiteProvisioningKey),
|
||||
logActionAudit(ActionsEnum.createSiteProvisioningKey),
|
||||
siteProvisioning.createSiteProvisioningKey
|
||||
);
|
||||
|
||||
authenticated.get(
|
||||
"/org/:orgId/site-provisioning-keys",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.siteProvisioningKeys),
|
||||
verifyOrgAccess,
|
||||
verifyUserHasAction(ActionsEnum.listSiteProvisioningKeys),
|
||||
siteProvisioning.listSiteProvisioningKeys
|
||||
);
|
||||
|
||||
authenticated.delete(
|
||||
"/org/:orgId/site-provisioning-key/:siteProvisioningKeyId",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.siteProvisioningKeys),
|
||||
verifyOrgAccess,
|
||||
verifySiteProvisioningKeyAccess,
|
||||
verifyUserHasAction(ActionsEnum.deleteSiteProvisioningKey),
|
||||
logActionAudit(ActionsEnum.deleteSiteProvisioningKey),
|
||||
siteProvisioning.deleteSiteProvisioningKey
|
||||
);
|
||||
|
||||
authenticated.patch(
|
||||
"/org/:orgId/site-provisioning-key/:siteProvisioningKeyId",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.siteProvisioningKeys),
|
||||
verifyOrgAccess,
|
||||
verifySiteProvisioningKeyAccess,
|
||||
verifyUserHasAction(ActionsEnum.updateSiteProvisioningKey),
|
||||
logActionAudit(ActionsEnum.updateSiteProvisioningKey),
|
||||
siteProvisioning.updateSiteProvisioningKey
|
||||
);
|
||||
|
||||
@@ -94,6 +94,25 @@ authenticated.get(
|
||||
logs.exportAccessAuditLogs
|
||||
);
|
||||
|
||||
authenticated.get(
|
||||
"/org/:orgId/logs/connection",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.connectionLogs),
|
||||
verifyApiKeyOrgAccess,
|
||||
verifyApiKeyHasAction(ActionsEnum.exportLogs),
|
||||
logs.queryConnectionAuditLogs
|
||||
);
|
||||
|
||||
authenticated.get(
|
||||
"/org/:orgId/logs/connection/export",
|
||||
verifyValidLicense,
|
||||
verifyValidSubscription(tierMatrix.logExport),
|
||||
verifyApiKeyOrgAccess,
|
||||
verifyApiKeyHasAction(ActionsEnum.exportLogs),
|
||||
logActionAudit(ActionsEnum.exportLogs),
|
||||
logs.exportConnectionAuditLogs
|
||||
);
|
||||
|
||||
authenticated.put(
|
||||
"/org/:orgId/idp/oidc",
|
||||
verifyValidLicense,
|
||||
|
||||
394
server/private/routers/newt/handleConnectionLogMessage.ts
Normal file
394
server/private/routers/newt/handleConnectionLogMessage.ts
Normal file
@@ -0,0 +1,394 @@
|
||||
import { db, logsDb } from "@server/db";
|
||||
import { MessageHandler } from "@server/routers/ws";
|
||||
import { connectionAuditLog, sites, Newt, clients, orgs } from "@server/db";
|
||||
import { and, eq, lt, inArray } from "drizzle-orm";
|
||||
import logger from "@server/logger";
|
||||
import { inflate } from "zlib";
|
||||
import { promisify } from "util";
|
||||
import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs";
|
||||
|
||||
const zlibInflate = promisify(inflate);
|
||||
|
||||
// Retry configuration for deadlock handling
|
||||
const MAX_RETRIES = 3;
|
||||
const BASE_DELAY_MS = 50;
|
||||
|
||||
// How often to flush accumulated connection log data to the database
|
||||
const FLUSH_INTERVAL_MS = 30_000; // 30 seconds
|
||||
|
||||
// Maximum number of records to buffer before forcing a flush
|
||||
const MAX_BUFFERED_RECORDS = 500;
|
||||
|
||||
// Maximum number of records to insert in a single batch
|
||||
const INSERT_BATCH_SIZE = 100;
|
||||
|
||||
interface ConnectionSessionData {
|
||||
sessionId: string;
|
||||
resourceId: number;
|
||||
sourceAddr: string;
|
||||
destAddr: string;
|
||||
protocol: string;
|
||||
startedAt: string; // ISO 8601 timestamp
|
||||
endedAt?: string; // ISO 8601 timestamp
|
||||
bytesTx?: number;
|
||||
bytesRx?: number;
|
||||
}
|
||||
|
||||
interface ConnectionLogRecord {
|
||||
sessionId: string;
|
||||
siteResourceId: number;
|
||||
orgId: string;
|
||||
siteId: number;
|
||||
clientId: number | null;
|
||||
userId: string | null;
|
||||
sourceAddr: string;
|
||||
destAddr: string;
|
||||
protocol: string;
|
||||
startedAt: number; // epoch seconds
|
||||
endedAt: number | null;
|
||||
bytesTx: number | null;
|
||||
bytesRx: number | null;
|
||||
}
|
||||
|
||||
// In-memory buffer of records waiting to be flushed
|
||||
let buffer: ConnectionLogRecord[] = [];
|
||||
|
||||
/**
|
||||
* Check if an error is a deadlock error
|
||||
*/
|
||||
function isDeadlockError(error: any): boolean {
|
||||
return (
|
||||
error?.code === "40P01" ||
|
||||
error?.cause?.code === "40P01" ||
|
||||
(error?.message && error.message.includes("deadlock"))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function with retry logic for deadlock handling
|
||||
*/
|
||||
async function withDeadlockRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
context: string
|
||||
): Promise<T> {
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error: any) {
|
||||
if (isDeadlockError(error) && attempt < MAX_RETRIES) {
|
||||
attempt++;
|
||||
const baseDelay = Math.pow(2, attempt - 1) * BASE_DELAY_MS;
|
||||
const jitter = Math.random() * baseDelay;
|
||||
const delay = baseDelay + jitter;
|
||||
logger.warn(
|
||||
`Deadlock detected in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress a base64-encoded zlib-compressed string into parsed JSON.
|
||||
*/
|
||||
async function decompressConnectionLog(
|
||||
compressed: string
|
||||
): Promise<ConnectionSessionData[]> {
|
||||
const compressedBuffer = Buffer.from(compressed, "base64");
|
||||
const decompressed = await zlibInflate(compressedBuffer);
|
||||
const jsonString = decompressed.toString("utf-8");
|
||||
const parsed = JSON.parse(jsonString);
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
throw new Error("Decompressed connection log data is not an array");
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an ISO 8601 timestamp string to epoch seconds.
|
||||
* Returns null if the input is falsy.
|
||||
*/
|
||||
function toEpochSeconds(isoString: string | undefined | null): number | null {
|
||||
if (!isoString) {
|
||||
return null;
|
||||
}
|
||||
const ms = new Date(isoString).getTime();
|
||||
if (isNaN(ms)) {
|
||||
return null;
|
||||
}
|
||||
return Math.floor(ms / 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush all buffered connection log records to the database.
|
||||
*
|
||||
* Swaps out the buffer before writing so that any records added during the
|
||||
* flush are captured in the new buffer rather than being lost. Entries that
|
||||
* fail to write are re-queued back into the buffer so they will be retried
|
||||
* on the next flush.
|
||||
*
|
||||
* This function is exported so that the application's graceful-shutdown
|
||||
* cleanup handler can call it before the process exits.
|
||||
*/
|
||||
export async function flushConnectionLogToDb(): Promise<void> {
|
||||
if (buffer.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Atomically swap out the buffer so new data keeps flowing in
|
||||
const snapshot = buffer;
|
||||
buffer = [];
|
||||
|
||||
logger.debug(
|
||||
`Flushing ${snapshot.length} connection log record(s) to the database`
|
||||
);
|
||||
|
||||
// Insert in batches to avoid overly large SQL statements
|
||||
for (let i = 0; i < snapshot.length; i += INSERT_BATCH_SIZE) {
|
||||
const batch = snapshot.slice(i, i + INSERT_BATCH_SIZE);
|
||||
|
||||
try {
|
||||
await withDeadlockRetry(async () => {
|
||||
await logsDb.insert(connectionAuditLog).values(batch);
|
||||
}, `flush connection log batch (${batch.length} records)`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to flush connection log batch of ${batch.length} records:`,
|
||||
error
|
||||
);
|
||||
|
||||
// Re-queue the failed batch so it is retried on the next flush
|
||||
buffer = [...batch, ...buffer];
|
||||
|
||||
// Cap buffer to prevent unbounded growth if DB is unreachable
|
||||
if (buffer.length > MAX_BUFFERED_RECORDS * 5) {
|
||||
const dropped = buffer.length - MAX_BUFFERED_RECORDS * 5;
|
||||
buffer = buffer.slice(0, MAX_BUFFERED_RECORDS * 5);
|
||||
logger.warn(
|
||||
`Connection log buffer overflow, dropped ${dropped} oldest records`
|
||||
);
|
||||
}
|
||||
|
||||
// Stop trying further batches from this snapshot — they'll be
|
||||
// picked up by the next flush via the re-queued records above
|
||||
const remaining = snapshot.slice(i + INSERT_BATCH_SIZE);
|
||||
if (remaining.length > 0) {
|
||||
buffer = [...remaining, ...buffer];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const flushTimer = setInterval(async () => {
|
||||
try {
|
||||
await flushConnectionLogToDb();
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
"Unexpected error during periodic connection log flush:",
|
||||
error
|
||||
);
|
||||
}
|
||||
}, FLUSH_INTERVAL_MS);
|
||||
|
||||
// Calling unref() means this timer will not keep the Node.js event loop alive
|
||||
// on its own — the process can still exit normally when there is no other work
|
||||
// left. The graceful-shutdown path will call flushConnectionLogToDb() explicitly
|
||||
// before process.exit(), so no data is lost.
|
||||
flushTimer.unref();
|
||||
|
||||
export async function cleanUpOldLogs(orgId: string, retentionDays: number) {
|
||||
const cutoffTimestamp = calculateCutoffTimestamp(retentionDays);
|
||||
|
||||
try {
|
||||
await logsDb
|
||||
.delete(connectionAuditLog)
|
||||
.where(
|
||||
and(
|
||||
lt(connectionAuditLog.startedAt, cutoffTimestamp),
|
||||
eq(connectionAuditLog.orgId, orgId)
|
||||
)
|
||||
);
|
||||
|
||||
// logger.debug(
|
||||
// `Cleaned up connection audit logs older than ${retentionDays} days`
|
||||
// );
|
||||
} catch (error) {
|
||||
logger.error("Error cleaning up old connection audit logs:", error);
|
||||
}
|
||||
}
|
||||
|
||||
export const handleConnectionLogMessage: MessageHandler = async (context) => {
|
||||
const { message, client } = context;
|
||||
const newt = client as Newt;
|
||||
|
||||
if (!newt) {
|
||||
logger.warn("Connection log received but no newt client in context");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!newt.siteId) {
|
||||
logger.warn("Connection log received but newt has no siteId");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!message.data?.compressed) {
|
||||
logger.warn("Connection log message missing compressed data");
|
||||
return;
|
||||
}
|
||||
|
||||
// Look up the org for this site
|
||||
const [site] = await db
|
||||
.select({ orgId: sites.orgId, orgSubnet: orgs.subnet })
|
||||
.from(sites)
|
||||
.innerJoin(orgs, eq(sites.orgId, orgs.orgId))
|
||||
.where(eq(sites.siteId, newt.siteId));
|
||||
|
||||
if (!site) {
|
||||
logger.warn(
|
||||
`Connection log received but site ${newt.siteId} not found in database`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const orgId = site.orgId;
|
||||
|
||||
// Extract the CIDR suffix (e.g. "/16") from the org subnet so we can
|
||||
// reconstruct the exact subnet string stored on each client record.
|
||||
const cidrSuffix = site.orgSubnet?.includes("/")
|
||||
? site.orgSubnet.substring(site.orgSubnet.indexOf("/"))
|
||||
: null;
|
||||
|
||||
let sessions: ConnectionSessionData[];
|
||||
try {
|
||||
sessions = await decompressConnectionLog(message.data.compressed);
|
||||
} catch (error) {
|
||||
logger.error("Failed to decompress connection log data:", error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (sessions.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug(`Sessions: ${JSON.stringify(sessions)}`)
|
||||
|
||||
// Build a map from sourceAddr → { clientId, userId } by querying clients
|
||||
// whose subnet field matches exactly. Client subnets are stored with the
|
||||
// org's CIDR suffix (e.g. "100.90.128.5/16"), so we reconstruct that from
|
||||
// each unique sourceAddr + the org's CIDR suffix and do a targeted IN query.
|
||||
const ipToClient = new Map<string, { clientId: number; userId: string | null }>();
|
||||
|
||||
if (cidrSuffix) {
|
||||
// Collect unique source addresses so we only query for what we need
|
||||
const uniqueSourceAddrs = new Set<string>();
|
||||
for (const session of sessions) {
|
||||
if (session.sourceAddr) {
|
||||
uniqueSourceAddrs.add(session.sourceAddr);
|
||||
}
|
||||
}
|
||||
|
||||
if (uniqueSourceAddrs.size > 0) {
|
||||
// Construct the exact subnet strings as stored in the DB
|
||||
const subnetQueries = Array.from(uniqueSourceAddrs).map(
|
||||
(addr) => {
|
||||
// Strip port if present (e.g. "100.90.128.1:38004" → "100.90.128.1")
|
||||
const ip = addr.includes(":") ? addr.split(":")[0] : addr;
|
||||
return `${ip}${cidrSuffix}`;
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug(`Subnet queries: ${JSON.stringify(subnetQueries)}`);
|
||||
|
||||
const matchedClients = await db
|
||||
.select({
|
||||
clientId: clients.clientId,
|
||||
userId: clients.userId,
|
||||
subnet: clients.subnet
|
||||
})
|
||||
.from(clients)
|
||||
.where(
|
||||
and(
|
||||
eq(clients.orgId, orgId),
|
||||
inArray(clients.subnet, subnetQueries)
|
||||
)
|
||||
);
|
||||
|
||||
for (const c of matchedClients) {
|
||||
const ip = c.subnet.split("/")[0];
|
||||
logger.debug(`Client ${c.clientId} subnet ${c.subnet} matches ${ip}`);
|
||||
ipToClient.set(ip, { clientId: c.clientId, userId: c.userId });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to DB records and add to the buffer
|
||||
for (const session of sessions) {
|
||||
// Validate required fields
|
||||
if (
|
||||
!session.sessionId ||
|
||||
!session.resourceId ||
|
||||
!session.sourceAddr ||
|
||||
!session.destAddr ||
|
||||
!session.protocol
|
||||
) {
|
||||
logger.debug(
|
||||
`Skipping connection log session with missing required fields: ${JSON.stringify(session)}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const startedAt = toEpochSeconds(session.startedAt);
|
||||
if (startedAt === null) {
|
||||
logger.debug(
|
||||
`Skipping connection log session with invalid startedAt: ${session.startedAt}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Match the source address to a client. The sourceAddr is the
|
||||
// client's IP on the WireGuard network, which corresponds to the IP
|
||||
// portion of the client's subnet CIDR (e.g. "100.90.128.5/24").
|
||||
// Strip port if present (e.g. "100.90.128.1:38004" → "100.90.128.1")
|
||||
const sourceIp = session.sourceAddr.includes(":") ? session.sourceAddr.split(":")[0] : session.sourceAddr;
|
||||
const clientInfo = ipToClient.get(sourceIp) ?? null;
|
||||
|
||||
|
||||
buffer.push({
|
||||
sessionId: session.sessionId,
|
||||
siteResourceId: session.resourceId,
|
||||
orgId,
|
||||
siteId: newt.siteId,
|
||||
clientId: clientInfo?.clientId ?? null,
|
||||
userId: clientInfo?.userId ?? null,
|
||||
sourceAddr: session.sourceAddr,
|
||||
destAddr: session.destAddr,
|
||||
protocol: session.protocol,
|
||||
startedAt,
|
||||
endedAt: toEpochSeconds(session.endedAt),
|
||||
bytesTx: session.bytesTx ?? null,
|
||||
bytesRx: session.bytesRx ?? null
|
||||
});
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`Buffered ${sessions.length} connection log session(s) from newt ${newt.newtId} (site ${newt.siteId})`
|
||||
);
|
||||
|
||||
// If the buffer has grown large enough, trigger an immediate flush
|
||||
if (buffer.length >= MAX_BUFFERED_RECORDS) {
|
||||
// Fire and forget — errors are handled inside flushConnectionLogToDb
|
||||
flushConnectionLogToDb().catch((error) => {
|
||||
logger.error(
|
||||
"Unexpected error during size-triggered connection log flush:",
|
||||
error
|
||||
);
|
||||
});
|
||||
}
|
||||
};
|
||||
1
server/private/routers/newt/index.ts
Normal file
1
server/private/routers/newt/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from "./handleConnectionLogMessage";
|
||||
@@ -0,0 +1,146 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
import { NextFunction, Request, Response } from "express";
|
||||
import { db, siteProvisioningKeyOrg, siteProvisioningKeys } from "@server/db";
|
||||
import HttpCode from "@server/types/HttpCode";
|
||||
import { z } from "zod";
|
||||
import { fromError } from "zod-validation-error";
|
||||
import createHttpError from "http-errors";
|
||||
import response from "@server/lib/response";
|
||||
import moment from "moment";
|
||||
import {
|
||||
generateId,
|
||||
generateIdFromEntropySize
|
||||
} from "@server/auth/sessions/app";
|
||||
import logger from "@server/logger";
|
||||
import { hashPassword } from "@server/auth/password";
|
||||
import type { CreateSiteProvisioningKeyResponse } from "@server/routers/siteProvisioning/types";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
orgId: z.string().nonempty()
|
||||
});
|
||||
|
||||
const bodySchema = z
|
||||
.strictObject({
|
||||
name: z.string().min(1).max(255),
|
||||
maxBatchSize: z.union([
|
||||
z.null(),
|
||||
z.coerce.number().int().positive().max(1_000_000)
|
||||
]),
|
||||
validUntil: z.string().max(255).optional()
|
||||
})
|
||||
.superRefine((data, ctx) => {
|
||||
const v = data.validUntil;
|
||||
if (v == null || v.trim() === "") {
|
||||
return;
|
||||
}
|
||||
if (Number.isNaN(Date.parse(v))) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "Invalid validUntil",
|
||||
path: ["validUntil"]
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type CreateSiteProvisioningKeyBody = z.infer<typeof bodySchema>;
|
||||
|
||||
export async function createSiteProvisioningKey(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<any> {
|
||||
const parsedParams = paramsSchema.safeParse(req.params);
|
||||
if (!parsedParams.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedParams.error).toString()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const parsedBody = bodySchema.safeParse(req.body);
|
||||
if (!parsedBody.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedBody.error).toString()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const { orgId } = parsedParams.data;
|
||||
const { name, maxBatchSize } = parsedBody.data;
|
||||
const vuRaw = parsedBody.data.validUntil;
|
||||
const validUntil =
|
||||
vuRaw == null || vuRaw.trim() === ""
|
||||
? null
|
||||
: new Date(Date.parse(vuRaw)).toISOString();
|
||||
|
||||
const siteProvisioningKeyId = `spk-${generateId(15)}`;
|
||||
const siteProvisioningKey = generateIdFromEntropySize(25);
|
||||
const siteProvisioningKeyHash = await hashPassword(siteProvisioningKey);
|
||||
const lastChars = siteProvisioningKey.slice(-4);
|
||||
const createdAt = moment().toISOString();
|
||||
const provisioningKey = `${siteProvisioningKeyId}.${siteProvisioningKey}`;
|
||||
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.insert(siteProvisioningKeys).values({
|
||||
siteProvisioningKeyId,
|
||||
name,
|
||||
siteProvisioningKeyHash,
|
||||
createdAt,
|
||||
lastChars,
|
||||
lastUsed: null,
|
||||
maxBatchSize,
|
||||
numUsed: 0,
|
||||
validUntil
|
||||
});
|
||||
|
||||
await trx.insert(siteProvisioningKeyOrg).values({
|
||||
siteProvisioningKeyId,
|
||||
orgId
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
return response<CreateSiteProvisioningKeyResponse>(res, {
|
||||
data: {
|
||||
siteProvisioningKeyId,
|
||||
orgId,
|
||||
name,
|
||||
siteProvisioningKey: provisioningKey,
|
||||
lastChars,
|
||||
createdAt,
|
||||
lastUsed: null,
|
||||
maxBatchSize,
|
||||
numUsed: 0,
|
||||
validUntil
|
||||
},
|
||||
success: true,
|
||||
error: false,
|
||||
message: "Site provisioning key created",
|
||||
status: HttpCode.CREATED
|
||||
});
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.INTERNAL_SERVER_ERROR,
|
||||
"Failed to create site provisioning key"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
import { Request, Response, NextFunction } from "express";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
db,
|
||||
siteProvisioningKeyOrg,
|
||||
siteProvisioningKeys
|
||||
} from "@server/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import response from "@server/lib/response";
|
||||
import HttpCode from "@server/types/HttpCode";
|
||||
import createHttpError from "http-errors";
|
||||
import logger from "@server/logger";
|
||||
import { fromError } from "zod-validation-error";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
siteProvisioningKeyId: z.string().nonempty(),
|
||||
orgId: z.string().nonempty()
|
||||
});
|
||||
|
||||
export async function deleteSiteProvisioningKey(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<any> {
|
||||
try {
|
||||
const parsedParams = paramsSchema.safeParse(req.params);
|
||||
if (!parsedParams.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedParams.error).toString()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const { siteProvisioningKeyId, orgId } = parsedParams.data;
|
||||
|
||||
const [row] = await db
|
||||
.select()
|
||||
.from(siteProvisioningKeys)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
)
|
||||
.innerJoin(
|
||||
siteProvisioningKeyOrg,
|
||||
and(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId
|
||||
),
|
||||
eq(siteProvisioningKeyOrg.orgId, orgId)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (!row) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.NOT_FOUND,
|
||||
`Site provisioning key with ID ${siteProvisioningKeyId} not found`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
await db.transaction(async (trx) => {
|
||||
await trx
|
||||
.delete(siteProvisioningKeyOrg)
|
||||
.where(
|
||||
and(
|
||||
eq(
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
),
|
||||
eq(siteProvisioningKeyOrg.orgId, orgId)
|
||||
)
|
||||
);
|
||||
|
||||
const siteProvisioningKeyOrgs = await trx
|
||||
.select()
|
||||
.from(siteProvisioningKeyOrg)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
);
|
||||
|
||||
if (siteProvisioningKeyOrgs.length === 0) {
|
||||
await trx
|
||||
.delete(siteProvisioningKeys)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return response(res, {
|
||||
data: null,
|
||||
success: true,
|
||||
error: false,
|
||||
message: "Site provisioning key deleted successfully",
|
||||
status: HttpCode.OK
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return next(
|
||||
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
|
||||
);
|
||||
}
|
||||
}
|
||||
17
server/private/routers/siteProvisioning/index.ts
Normal file
17
server/private/routers/siteProvisioning/index.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
export * from "./createSiteProvisioningKey";
|
||||
export * from "./listSiteProvisioningKeys";
|
||||
export * from "./deleteSiteProvisioningKey";
|
||||
export * from "./updateSiteProvisioningKey";
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
import {
|
||||
db,
|
||||
siteProvisioningKeyOrg,
|
||||
siteProvisioningKeys
|
||||
} from "@server/db";
|
||||
import logger from "@server/logger";
|
||||
import HttpCode from "@server/types/HttpCode";
|
||||
import response from "@server/lib/response";
|
||||
import { NextFunction, Request, Response } from "express";
|
||||
import createHttpError from "http-errors";
|
||||
import { z } from "zod";
|
||||
import { fromError } from "zod-validation-error";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { ListSiteProvisioningKeysResponse } from "@server/routers/siteProvisioning/types";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
orgId: z.string().nonempty()
|
||||
});
|
||||
|
||||
const querySchema = z.object({
|
||||
limit: z
|
||||
.string()
|
||||
.optional()
|
||||
.default("1000")
|
||||
.transform(Number)
|
||||
.pipe(z.int().positive()),
|
||||
offset: z
|
||||
.string()
|
||||
.optional()
|
||||
.default("0")
|
||||
.transform(Number)
|
||||
.pipe(z.int().nonnegative())
|
||||
});
|
||||
|
||||
function querySiteProvisioningKeys(orgId: string) {
|
||||
return db
|
||||
.select({
|
||||
siteProvisioningKeyId:
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
orgId: siteProvisioningKeyOrg.orgId,
|
||||
lastChars: siteProvisioningKeys.lastChars,
|
||||
createdAt: siteProvisioningKeys.createdAt,
|
||||
name: siteProvisioningKeys.name,
|
||||
lastUsed: siteProvisioningKeys.lastUsed,
|
||||
maxBatchSize: siteProvisioningKeys.maxBatchSize,
|
||||
numUsed: siteProvisioningKeys.numUsed,
|
||||
validUntil: siteProvisioningKeys.validUntil
|
||||
})
|
||||
.from(siteProvisioningKeyOrg)
|
||||
.innerJoin(
|
||||
siteProvisioningKeys,
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId
|
||||
)
|
||||
)
|
||||
.where(eq(siteProvisioningKeyOrg.orgId, orgId));
|
||||
}
|
||||
|
||||
export async function listSiteProvisioningKeys(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<any> {
|
||||
try {
|
||||
const parsedParams = paramsSchema.safeParse(req.params);
|
||||
if (!parsedParams.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedParams.error)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const parsedQuery = querySchema.safeParse(req.query);
|
||||
if (!parsedQuery.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedQuery.error)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const { orgId } = parsedParams.data;
|
||||
const { limit, offset } = parsedQuery.data;
|
||||
|
||||
const siteProvisioningKeysList = await querySiteProvisioningKeys(orgId)
|
||||
.limit(limit)
|
||||
.offset(offset);
|
||||
|
||||
return response<ListSiteProvisioningKeysResponse>(res, {
|
||||
data: {
|
||||
siteProvisioningKeys: siteProvisioningKeysList,
|
||||
pagination: {
|
||||
total: siteProvisioningKeysList.length,
|
||||
limit,
|
||||
offset
|
||||
}
|
||||
},
|
||||
success: true,
|
||||
error: false,
|
||||
message: "Site provisioning keys retrieved successfully",
|
||||
status: HttpCode.OK
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return next(
|
||||
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
/*
|
||||
* This file is part of a proprietary work.
|
||||
*
|
||||
* Copyright (c) 2025 Fossorial, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This file is licensed under the Fossorial Commercial License.
|
||||
* You may not use this file except in compliance with the License.
|
||||
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
|
||||
*
|
||||
* This file is not licensed under the AGPLv3.
|
||||
*/
|
||||
|
||||
import { Request, Response, NextFunction } from "express";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
db,
|
||||
siteProvisioningKeyOrg,
|
||||
siteProvisioningKeys
|
||||
} from "@server/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import response from "@server/lib/response";
|
||||
import HttpCode from "@server/types/HttpCode";
|
||||
import createHttpError from "http-errors";
|
||||
import logger from "@server/logger";
|
||||
import { fromError } from "zod-validation-error";
|
||||
import type { UpdateSiteProvisioningKeyResponse } from "@server/routers/siteProvisioning/types";
|
||||
|
||||
const paramsSchema = z.object({
|
||||
siteProvisioningKeyId: z.string().nonempty(),
|
||||
orgId: z.string().nonempty()
|
||||
});
|
||||
|
||||
const bodySchema = z
|
||||
.strictObject({
|
||||
maxBatchSize: z
|
||||
.union([
|
||||
z.null(),
|
||||
z.coerce.number().int().positive().max(1_000_000)
|
||||
])
|
||||
.optional(),
|
||||
validUntil: z.string().max(255).optional()
|
||||
})
|
||||
.superRefine((data, ctx) => {
|
||||
if (
|
||||
data.maxBatchSize === undefined &&
|
||||
data.validUntil === undefined
|
||||
) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "Provide maxBatchSize and/or validUntil",
|
||||
path: ["maxBatchSize"]
|
||||
});
|
||||
}
|
||||
const v = data.validUntil;
|
||||
if (v == null || v.trim() === "") {
|
||||
return;
|
||||
}
|
||||
if (Number.isNaN(Date.parse(v))) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "Invalid validUntil",
|
||||
path: ["validUntil"]
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type UpdateSiteProvisioningKeyBody = z.infer<typeof bodySchema>;
|
||||
|
||||
export async function updateSiteProvisioningKey(
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<any> {
|
||||
try {
|
||||
const parsedParams = paramsSchema.safeParse(req.params);
|
||||
if (!parsedParams.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedParams.error).toString()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const parsedBody = bodySchema.safeParse(req.body);
|
||||
if (!parsedBody.success) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.BAD_REQUEST,
|
||||
fromError(parsedBody.error).toString()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const { siteProvisioningKeyId, orgId } = parsedParams.data;
|
||||
const body = parsedBody.data;
|
||||
|
||||
const [row] = await db
|
||||
.select()
|
||||
.from(siteProvisioningKeys)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
)
|
||||
.innerJoin(
|
||||
siteProvisioningKeyOrg,
|
||||
and(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyOrg.siteProvisioningKeyId
|
||||
),
|
||||
eq(siteProvisioningKeyOrg.orgId, orgId)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (!row) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.NOT_FOUND,
|
||||
`Site provisioning key with ID ${siteProvisioningKeyId} not found`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const setValues: {
|
||||
maxBatchSize?: number | null;
|
||||
validUntil?: string | null;
|
||||
} = {};
|
||||
if (body.maxBatchSize !== undefined) {
|
||||
setValues.maxBatchSize = body.maxBatchSize;
|
||||
}
|
||||
if (body.validUntil !== undefined) {
|
||||
setValues.validUntil =
|
||||
body.validUntil.trim() === ""
|
||||
? null
|
||||
: new Date(Date.parse(body.validUntil)).toISOString();
|
||||
}
|
||||
|
||||
await db
|
||||
.update(siteProvisioningKeys)
|
||||
.set(setValues)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
);
|
||||
|
||||
const [updated] = await db
|
||||
.select({
|
||||
siteProvisioningKeyId:
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
name: siteProvisioningKeys.name,
|
||||
lastChars: siteProvisioningKeys.lastChars,
|
||||
createdAt: siteProvisioningKeys.createdAt,
|
||||
lastUsed: siteProvisioningKeys.lastUsed,
|
||||
maxBatchSize: siteProvisioningKeys.maxBatchSize,
|
||||
numUsed: siteProvisioningKeys.numUsed,
|
||||
validUntil: siteProvisioningKeys.validUntil
|
||||
})
|
||||
.from(siteProvisioningKeys)
|
||||
.where(
|
||||
eq(
|
||||
siteProvisioningKeys.siteProvisioningKeyId,
|
||||
siteProvisioningKeyId
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (!updated) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.INTERNAL_SERVER_ERROR,
|
||||
"Failed to load updated site provisioning key"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
return response<UpdateSiteProvisioningKeyResponse>(res, {
|
||||
data: {
|
||||
...updated,
|
||||
orgId
|
||||
},
|
||||
success: true,
|
||||
error: false,
|
||||
message: "Site provisioning key updated successfully",
|
||||
status: HttpCode.OK
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return next(
|
||||
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -18,10 +18,12 @@ import {
|
||||
} from "#private/routers/remoteExitNode";
|
||||
import { MessageHandler } from "@server/routers/ws";
|
||||
import { build } from "@server/build";
|
||||
import { handleConnectionLogMessage } from "#dynamic/routers/newt";
|
||||
|
||||
export const messageHandlers: Record<string, MessageHandler> = {
|
||||
"remoteExitNode/register": handleRemoteExitNodeRegisterMessage,
|
||||
"remoteExitNode/ping": handleRemoteExitNodePingMessage
|
||||
"remoteExitNode/ping": handleRemoteExitNodePingMessage,
|
||||
"newt/access-log": handleConnectionLogMessage,
|
||||
};
|
||||
|
||||
if (build != "saas") {
|
||||
|
||||
Reference in New Issue
Block a user