pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-02-27 15:26:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into dev

Browse Source
This commit is contained in:
Owen
2026-02-25 10:46:28 -08:00
parent 767284408a 2ede0d498a
commit 5b7f893ad7
31 changed files with 729 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
server/routers/auditLogs/queryRequestAnalytics.ts
View File

@@ -1,4 +1,4 @@
import { db, requestAuditLog, driver, primaryDb } from "@server/db";
import { logsDb, requestAuditLog, driver, primaryLogsDb } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
@@ -74,12 +74,12 @@ async function query(query: Q) {
);
}
const [all] = await primaryDb
const [all] = await primaryLogsDb
.select({ total: count() })
.from(requestAuditLog)
.where(baseConditions);
const [blocked] = await primaryDb
const [blocked] = await primaryLogsDb
.select({ total: count() })
.from(requestAuditLog)
.where(and(baseConditions, eq(requestAuditLog.action, false)));
@@ -90,7 +90,7 @@ async function query(query: Q) {
const DISTINCT_LIMIT = 500;
const requestsPerCountry = await primaryDb
const requestsPerCountry = await primaryLogsDb
.selectDistinct({
code: requestAuditLog.location,
count: totalQ
@@ -118,7 +118,7 @@ async function query(query: Q) {
const booleanTrue = driver === "pg" ? sql`true` : sql`1`;
const booleanFalse = driver === "pg" ? sql`false` : sql`0`;
const requestsPerDay = await primaryDb
const requestsPerDay = await primaryLogsDb
.select({
day: groupByDayFunction.as("day"),
allowedCount:

98
server/routers/auditLogs/queryRequestAuditLog.ts
View File

@@ -1,8 +1,8 @@
import { db, primaryDb, requestAuditLog, resources } from "@server/db";
import { logsDb, primaryLogsDb, requestAuditLog, resources, db, primaryDb } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
import { eq, gt, lt, and, count, desc } from "drizzle-orm";
import { eq, gt, lt, and, count, desc, inArray } from "drizzle-orm";
import { OpenAPITags } from "@server/openApi";
import { z } from "zod";
import createHttpError from "http-errors";
@@ -107,7 +107,7 @@ function getWhere(data: Q) {
}
export function queryRequest(data: Q) {
return primaryDb
return primaryLogsDb
.select({
id: requestAuditLog.id,
timestamp: requestAuditLog.timestamp,
@@ -129,21 +129,49 @@ export function queryRequest(data: Q) {
host: requestAuditLog.host,
path: requestAuditLog.path,
method: requestAuditLog.method,
tls: requestAuditLog.tls,
resourceName: resources.name,
resourceNiceId: resources.niceId
tls: requestAuditLog.tls
})
.from(requestAuditLog)
.leftJoin(
resources,
eq(requestAuditLog.resourceId, resources.resourceId)
) // TODO: Is this efficient?
.where(getWhere(data))
.orderBy(desc(requestAuditLog.timestamp));
}
async function enrichWithResourceDetails(logs: Awaited<ReturnType<typeof queryRequest>>) {
// If logs database is the same as main database, we can do a join
// Otherwise, we need to fetch resource details separately
const resourceIds = logs
.map(log => log.resourceId)
.filter((id): id is number => id !== null && id !== undefined);
if (resourceIds.length === 0) {
return logs.map(log => ({ ...log, resourceName: null, resourceNiceId: null }));
}
// Fetch resource details from main database
const resourceDetails = await primaryDb
.select({
resourceId: resources.resourceId,
name: resources.name,
niceId: resources.niceId
})
.from(resources)
.where(inArray(resources.resourceId, resourceIds));
// Create a map for quick lookup
const resourceMap = new Map(
resourceDetails.map(r => [r.resourceId, { name: r.name, niceId: r.niceId }])
);
// Enrich logs with resource details
return logs.map(log => ({
...log,
resourceName: log.resourceId ? resourceMap.get(log.resourceId)?.name ?? null : null,
resourceNiceId: log.resourceId ? resourceMap.get(log.resourceId)?.niceId ?? null : null
}));
}
export function countRequestQuery(data: Q) {
const countQuery = primaryDb
const countQuery = primaryLogsDb
.select({ count: count() })
.from(requestAuditLog)
.where(getWhere(data));
@@ -185,36 +213,31 @@ async function queryUniqueFilterAttributes(
uniquePaths,
uniqueResources
] = await Promise.all([
primaryDb
primaryLogsDb
.selectDistinct({ actor: requestAuditLog.actor })
.from(requestAuditLog)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
primaryDb
primaryLogsDb
.selectDistinct({ locations: requestAuditLog.location })
.from(requestAuditLog)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
primaryDb
primaryLogsDb
.selectDistinct({ hosts: requestAuditLog.host })
.from(requestAuditLog)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
primaryDb
primaryLogsDb
.selectDistinct({ paths: requestAuditLog.path })
.from(requestAuditLog)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
primaryDb
primaryLogsDb
.selectDistinct({
id: requestAuditLog.resourceId,
name: resources.name
id: requestAuditLog.resourceId
})
.from(requestAuditLog)
.leftJoin(
resources,
eq(requestAuditLog.resourceId, resources.resourceId)
)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1)
]);
@@ -231,13 +254,33 @@ async function queryUniqueFilterAttributes(
// throw new Error("Too many distinct filter attributes to retrieve. Please refine your time range.");
// }
// Fetch resource names from main database for the unique resource IDs
const resourceIds = uniqueResources
.map(row => row.id)
.filter((id): id is number => id !== null);
let resourcesWithNames: Array<{ id: number; name: string | null }> = [];
if (resourceIds.length > 0) {
const resourceDetails = await primaryDb
.select({
resourceId: resources.resourceId,
name: resources.name
})
.from(resources)
.where(inArray(resources.resourceId, resourceIds));
resourcesWithNames = resourceDetails.map(r => ({
id: r.resourceId,
name: r.name
}));
}
return {
actors: uniqueActors
.map((row) => row.actor)
.filter((actor): actor is string => actor !== null),
resources: uniqueResources.filter(
(row): row is { id: number; name: string | null } => row.id !== null
),
resources: resourcesWithNames,
locations: uniqueLocations
.map((row) => row.locations)
.filter((location): location is string => location !== null),
@@ -280,7 +323,10 @@ export async function queryRequestAuditLogs(
const baseQuery = queryRequest(data);
const log = await baseQuery.limit(data.limit).offset(data.offset);
const logsRaw = await baseQuery.limit(data.limit).offset(data.offset);
// Enrich with resource details (handles cross-database scenario)
const log = await enrichWithResourceDetails(logsRaw);
const totalCountResult = await countRequestQuery(data);
const totalCount = totalCountResult[0].count;

10
server/routers/badger/logRequestAudit.ts
View File

@@ -1,4 +1,4 @@
import { db, orgs, requestAuditLog } from "@server/db";
import { logsDb, primaryLogsDb, db, orgs, requestAuditLog } from "@server/db";
import logger from "@server/logger";
import { and, eq, lt, sql } from "drizzle-orm";
import cache from "@server/lib/cache";
@@ -69,7 +69,7 @@ async function flushAuditLogs() {
try {
// Use a transaction to ensure all inserts succeed or fail together
// This prevents index corruption from partial writes
await db.transaction(async (tx) => {
await logsDb.transaction(async (tx) => {
// Batch insert logs in groups of 25 to avoid overwhelming the database
const BATCH_DB_SIZE = 25;
for (let i = 0; i < logsToWrite.length; i += BATCH_DB_SIZE) {
@@ -130,7 +130,7 @@ export async function shutdownAuditLogger() {
async function getRetentionDays(orgId: string): Promise<number> {
// check cache first
const cached = cache.get<number>(`org_${orgId}_retentionDays`);
const cached = await cache.get<number>(`org_${orgId}_retentionDays`);
if (cached !== undefined) {
return cached;
}
@@ -149,7 +149,7 @@ async function getRetentionDays(orgId: string): Promise<number> {
}
// store the result in cache
cache.set(
await cache.set(
`org_${orgId}_retentionDays`,
org.settingsLogRetentionDaysRequest,
300
@@ -162,7 +162,7 @@ export async function cleanUpOldLogs(orgId: string, retentionDays: number) {
const cutoffTimestamp = calculateCutoffTimestamp(retentionDays);
try {
await db
await logsDb
.delete(requestAuditLog)
.where(
and(

30
server/routers/badger/verifySession.ts
View File

@@ -37,7 +37,7 @@ import {
enforceResourceSessionLength
} from "#dynamic/lib/checkOrgAccessPolicy";
import { logRequestAudit } from "./logRequestAudit";
import cache from "@server/lib/cache";
import { localCache } from "@server/lib/cache";
import { APP_VERSION } from "@server/lib/consts";
import { isSubscribed } from "#dynamic/lib/isSubscribed";
import { tierMatrix } from "@server/lib/billing/tierMatrix";
@@ -137,7 +137,7 @@ export async function verifyResourceSession(
headerAuthExtendedCompatibility: ResourceHeaderAuthExtendedCompatibility | null;
org: Org;
}
| undefined = cache.get(resourceCacheKey);
| undefined = localCache.get(resourceCacheKey);
if (!resourceData) {
const result = await getResourceByDomain(cleanHost);
@@ -161,7 +161,7 @@ export async function verifyResourceSession(
}
resourceData = result;
cache.set(resourceCacheKey, resourceData, 5);
localCache.set(resourceCacheKey, resourceData, 5);
}
const {
@@ -405,7 +405,7 @@ export async function verifyResourceSession(
// check for HTTP Basic Auth header
const clientHeaderAuthKey = `headerAuth:${clientHeaderAuth}`;
if (headerAuth && clientHeaderAuth) {
if (cache.get(clientHeaderAuthKey)) {
if (localCache.get(clientHeaderAuthKey)) {
logger.debug(
"Resource allowed because header auth is valid (cached)"
);
@@ -428,7 +428,7 @@ export async function verifyResourceSession(
headerAuth.headerAuthHash
)
) {
cache.set(clientHeaderAuthKey, clientHeaderAuth, 5);
localCache.set(clientHeaderAuthKey, clientHeaderAuth, 5);
logger.debug("Resource allowed because header auth is valid");
logRequestAudit(
@@ -520,7 +520,7 @@ export async function verifyResourceSession(
if (resourceSessionToken) {
const sessionCacheKey = `session:${resourceSessionToken}`;
let resourceSession: any = cache.get(sessionCacheKey);
let resourceSession: any = localCache.get(sessionCacheKey);
if (!resourceSession) {
const result = await validateResourceSessionToken(
@@ -529,7 +529,7 @@ export async function verifyResourceSession(
);
resourceSession = result?.resourceSession;
cache.set(sessionCacheKey, resourceSession, 5);
localCache.set(sessionCacheKey, resourceSession, 5);
}
if (resourceSession?.isRequestToken) {
@@ -662,7 +662,7 @@ export async function verifyResourceSession(
}:${resource.resourceId}`;
let allowedUserData: BasicUserData | null | undefined =
cache.get(userAccessCacheKey);
localCache.get(userAccessCacheKey);
if (allowedUserData === undefined) {
allowedUserData = await isUserAllowedToAccessResource(
@@ -671,7 +671,7 @@ export async function verifyResourceSession(
resourceData.org
);
cache.set(userAccessCacheKey, allowedUserData, 5);
localCache.set(userAccessCacheKey, allowedUserData, 5);
}
if (
@@ -974,11 +974,11 @@ async function checkRules(
): Promise<"ACCEPT" | "DROP" | "PASS" | undefined> {
const ruleCacheKey = `rules:${resourceId}`;
let rules: ResourceRule[] | undefined = cache.get(ruleCacheKey);
let rules: ResourceRule[] | undefined = localCache.get(ruleCacheKey);
if (!rules) {
rules = await getResourceRules(resourceId);
cache.set(ruleCacheKey, rules, 5);
localCache.set(ruleCacheKey, rules, 5);
}
if (rules.length === 0) {
@@ -1208,13 +1208,13 @@ async function isIpInAsn(
async function getAsnFromIp(ip: string): Promise<number | undefined> {
const asnCacheKey = `asn:${ip}`;
let cachedAsn: number | undefined = cache.get(asnCacheKey);
let cachedAsn: number | undefined = localCache.get(asnCacheKey);
if (!cachedAsn) {
cachedAsn = await getAsnForIp(ip); // do it locally
// Cache for longer since IP ASN doesn't change frequently
if (cachedAsn) {
cache.set(asnCacheKey, cachedAsn, 300); // 5 minutes
localCache.set(asnCacheKey, cachedAsn, 300); // 5 minutes
}
}
@@ -1224,14 +1224,14 @@ async function getAsnFromIp(ip: string): Promise<number | undefined> {
async function getCountryCodeFromIp(ip: string): Promise<string | undefined> {
const geoIpCacheKey = `geoip:${ip}`;
let cachedCountryCode: string | undefined = cache.get(geoIpCacheKey);
let cachedCountryCode: string | undefined = localCache.get(geoIpCacheKey);
if (!cachedCountryCode) {
cachedCountryCode = await getCountryCodeForIp(ip); // do it locally
// Only cache successful lookups to avoid filling cache with undefined values
if (cachedCountryCode) {
// Cache for longer since IP geolocation doesn't change frequently
cache.set(geoIpCacheKey, cachedCountryCode, 300); // 5 minutes
localCache.set(geoIpCacheKey, cachedCountryCode, 300); // 5 minutes
}
}

6
server/routers/newt/handleSocketMessages.ts
View File

@@ -24,8 +24,8 @@ export const handleDockerStatusMessage: MessageHandler = async (context) => {
if (available) {
logger.info(`Newt ${newt.newtId} has Docker socket access`);
cache.set(`${newt.newtId}:socketPath`, socketPath, 0);
cache.set(`${newt.newtId}:isAvailable`, available, 0);
await cache.set(`${newt.newtId}:socketPath`, socketPath, 0);
await cache.set(`${newt.newtId}:isAvailable`, available, 0);
} else {
logger.warn(`Newt ${newt.newtId} does not have Docker socket access`);
}
@@ -54,7 +54,7 @@ export const handleDockerContainersMessage: MessageHandler = async (
);
if (containers && containers.length > 0) {
cache.set(`${newt.newtId}:dockerContainers`, containers, 0);
await cache.set(`${newt.newtId}:dockerContainers`, containers, 0);
} else {
logger.warn(`Newt ${newt.newtId} does not have Docker containers`);
}

58
server/routers/olm/getOlmToken.ts
View File

@@ -1,4 +1,7 @@
import { generateSessionToken } from "@server/auth/sessions/app";
import {
generateSessionToken,
validateSessionToken
} from "@server/auth/sessions/app";
import {
clients,
db,
@@ -26,8 +29,9 @@ import { APP_VERSION } from "@server/lib/consts";
export const olmGetTokenBodySchema = z.object({
olmId: z.string(),
secret: z.string(),
token: z.string().optional(),
secret: z.string().optional(),
userToken: z.string().optional(),
token: z.string().optional(), // this is the olm token
orgId: z.string().optional()
});
@@ -49,7 +53,7 @@ export async function getOlmToken(
);
}
const { olmId, secret, token, orgId } = parsedBody.data;
const { olmId, secret, token, orgId, userToken } = parsedBody.data;
try {
if (token) {
@@ -84,19 +88,45 @@ export async function getOlmToken(
);
}
const validSecret = await verifyPassword(
secret,
existingOlm.secretHash
);
if (!validSecret) {
if (config.getRawConfig().app.log_failed_attempts) {
logger.info(
`Olm id or secret is incorrect. Olm: ID ${olmId}. IP: ${req.ip}.`
if (userToken) {
const { session: userSession, user } =
await validateSessionToken(userToken);
if (!userSession || !user) {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Invalid user token")
);
}
if (user.userId !== existingOlm.userId) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
"User token does not match olm"
)
);
}
} else if (secret) {
// this is for backward compatibility, we want to move towards userToken but some old clients may still be using secret so we will support both for now
const validSecret = await verifyPassword(
secret,
existingOlm.secretHash
);
if (!validSecret) {
if (config.getRawConfig().app.log_failed_attempts) {
logger.info(
`Olm id or secret is incorrect. Olm: ID ${olmId}. IP: ${req.ip}.`
);
}
return next(
createHttpError(HttpCode.BAD_REQUEST, "Secret is incorrect")
);
}
} else {
return next(
createHttpError(HttpCode.BAD_REQUEST, "Secret is incorrect")
createHttpError(
HttpCode.BAD_REQUEST,
"Either secret or userToken is required"
)
);
}

6
server/routers/org/updateOrg.ts
View File

@@ -194,9 +194,9 @@ export async function updateOrg(
}
// invalidate the cache for all of the orgs retention days
cache.del(`org_${orgId}_retentionDays`);
cache.del(`org_${orgId}_actionDays`);
cache.del(`org_${orgId}_accessDays`);
await cache.del(`org_${orgId}_retentionDays`);
await cache.del(`org_${orgId}_actionDays`);
await cache.del(`org_${orgId}_accessDays`);
return response(res, {
data: updatedOrg[0],

4
server/routers/site/listSites.ts
View File

@@ -23,7 +23,7 @@ import { fromError } from "zod-validation-error";
async function getLatestNewtVersion(): Promise<string | null> {
try {
const cachedVersion = cache.get<string>("latestNewtVersion");
const cachedVersion = await cache.get<string>("latestNewtVersion");
if (cachedVersion) {
return cachedVersion;
}
@@ -55,7 +55,7 @@ async function getLatestNewtVersion(): Promise<string | null> {
tags = tags.filter((version) => !version.name.includes("rc"));
const latestVersion = tags[0].name;
cache.set("latestNewtVersion", latestVersion);
await cache.set("latestNewtVersion", latestVersion);
return latestVersion;
} catch (error: any) {

12
server/routers/site/socketIntegration.ts
View File

@@ -150,7 +150,7 @@ async function triggerFetch(siteId: number) {
// clear the cache for this Newt ID so that the site has to keep asking for the containers
// this is to ensure that the site always gets the latest data
cache.del(`${newt.newtId}:dockerContainers`);
await cache.del(`${newt.newtId}:dockerContainers`);
return { siteId, newtId: newt.newtId };
}
@@ -158,7 +158,7 @@ async function triggerFetch(siteId: number) {
async function queryContainers(siteId: number) {
const { newt } = await getSiteAndNewt(siteId);
const result = cache.get(`${newt.newtId}:dockerContainers`) as Container[];
const result = await cache.get<Container[]>(`${newt.newtId}:dockerContainers`);
if (!result) {
throw createHttpError(
HttpCode.TOO_EARLY,
@@ -173,7 +173,7 @@ async function isDockerAvailable(siteId: number): Promise<boolean> {
const { newt } = await getSiteAndNewt(siteId);
const key = `${newt.newtId}:isAvailable`;
const isAvailable = cache.get(key);
const isAvailable = await cache.get(key);
return !!isAvailable;
}
@@ -186,9 +186,11 @@ async function getDockerStatus(
const keys = ["isAvailable", "socketPath"];
const mappedKeys = keys.map((x) => `${newt.newtId}:${x}`);
const values = await cache.mget<boolean | string>(mappedKeys);
const result = {
isAvailable: cache.get(mappedKeys[0]) as boolean,
socketPath: cache.get(mappedKeys[1]) as string | undefined
isAvailable: values[0] as boolean,
socketPath: values[1] as string | undefined
};
return result;

4
server/routers/user/inviteUser.ts
View File

@@ -191,7 +191,7 @@ export async function inviteUser(
}
if (existingInvite.length) {
const attempts = cache.get<number>(email) || 0;
const attempts = (await cache.get<number>(email)) || 0;
if (attempts >= 3) {
return next(
createHttpError(
@@ -201,7 +201,7 @@ export async function inviteUser(
);
}
cache.set(email, attempts + 1);
await cache.set(email, attempts + 1);
const inviteId = existingInvite[0].inviteId; // Retrieve the original inviteId
const token = generateRandomString(