pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-04-13 13:26:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Logging and http working

Browse Source
This commit is contained in:
Owen
2026-04-12 15:08:17 -07:00
parent 0cf385b718
commit 789b991c56
8 changed files with 212 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/db/pg/schema/schema.ts
View File

@@ -1004,6 +1004,7 @@ export const requestAuditLog = pgTable(
actor: text("actor"),
actorId: text("actorId"),
resourceId: integer("resourceId"),
siteResourceId: integer("siteResourceId"),
ip: text("ip"),
location: text("location"),
userAgent: text("userAgent"),

1
server/db/sqlite/schema/schema.ts
View File

@@ -1104,6 +1104,7 @@ export const requestAuditLog = sqliteTable(
actor: text("actor"),
actorId: text("actorId"),
resourceId: integer("resourceId"),
siteResourceId: integer("siteResourceId"),
ip: text("ip"),
location: text("location"),
userAgent: text("userAgent"),

16
server/private/routers/newt/handleConnectionLogMessage.ts
View File

@@ -92,9 +92,14 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => {
return;
}
// Look up the org for this site
// Look up the org for this site and check retention settings
const [site] = await db
.select({ orgId: sites.orgId, orgSubnet: orgs.subnet })
.select({
orgId: sites.orgId,
orgSubnet: orgs.subnet,
settingsLogRetentionDaysConnection:
orgs.settingsLogRetentionDaysConnection
})
.from(sites)
.innerJoin(orgs, eq(sites.orgId, orgs.orgId))
.where(eq(sites.siteId, newt.siteId));
@@ -108,6 +113,13 @@ export const handleConnectionLogMessage: MessageHandler = async (context) => {
const orgId = site.orgId;
if (site.settingsLogRetentionDaysConnection === 0) {
logger.debug(
`Connection log retention is disabled for org ${orgId}, skipping`
);
return;
}
// Extract the CIDR suffix (e.g. "/16") from the org subnet so we can
// reconstruct the exact subnet string stored on each client record.
const cidrSuffix = site.orgSubnet?.includes("/")

80
server/private/routers/newt/handleRequestLogMessage.ts
View File

@@ -13,12 +13,13 @@
import { db } from "@server/db";
import { MessageHandler } from "@server/routers/ws";
import { sites, Newt, orgs } from "@server/db";
import { eq } from "drizzle-orm";
import { sites, Newt, orgs, clients, clientSitesAssociationsCache } from "@server/db";
import { and, eq, inArray } from "drizzle-orm";
import logger from "@server/logger";
import { inflate } from "zlib";
import { promisify } from "util";
import { logRequestAudit } from "@server/routers/badger/logRequestAudit";
import { getCountryCodeForIp } from "@server/lib/geoip";
export async function flushRequestLogToDb(): Promise<void> {
return;
@@ -81,6 +82,7 @@ export const handleRequestLogMessage: MessageHandler = async (context) => {
const [site] = await db
.select({
orgId: sites.orgId,
orgSubnet: orgs.subnet,
settingsLogRetentionDaysRequest:
orgs.settingsLogRetentionDaysRequest
})
@@ -118,6 +120,61 @@ export const handleRequestLogMessage: MessageHandler = async (context) => {
logger.debug(`Request log entries: ${JSON.stringify(entries)}`);
// Build a map from sourceIp → external endpoint string by joining clients
// with clientSitesAssociationsCache. The endpoint is the real-world IP:port
// of the client device and is used for GeoIP lookup.
const ipToEndpoint = new Map<string, string>();
const cidrSuffix = site.orgSubnet?.includes("/")
? site.orgSubnet.substring(site.orgSubnet.indexOf("/"))
: null;
if (cidrSuffix) {
const uniqueSourceAddrs = new Set<string>();
for (const entry of entries) {
if (entry.sourceAddr) {
uniqueSourceAddrs.add(entry.sourceAddr);
}
}
if (uniqueSourceAddrs.size > 0) {
const subnetQueries = Array.from(uniqueSourceAddrs).map((addr) => {
const ip = addr.includes(":") ? addr.split(":")[0] : addr;
return `${ip}${cidrSuffix}`;
});
const matchedClients = await db
.select({
subnet: clients.subnet,
endpoint: clientSitesAssociationsCache.endpoint
})
.from(clients)
.innerJoin(
clientSitesAssociationsCache,
and(
eq(
clientSitesAssociationsCache.clientId,
clients.clientId
),
eq(clientSitesAssociationsCache.siteId, newt.siteId)
)
)
.where(
and(
eq(clients.orgId, orgId),
inArray(clients.subnet, subnetQueries)
)
);
for (const c of matchedClients) {
if (c.endpoint) {
const ip = c.subnet.split("/")[0];
ipToEndpoint.set(ip, c.endpoint);
}
}
}
}
for (const entry of entries) {
if (
!entry.requestId ||
@@ -141,12 +198,27 @@ export const handleRequestLogMessage: MessageHandler = async (context) => {
entry.path +
(entry.rawQuery ? "?" + entry.rawQuery : "");
// Resolve the client's external endpoint for GeoIP lookup.
// sourceAddr is the WireGuard IP (possibly ip:port), so strip the port.
const sourceIp = entry.sourceAddr.includes(":")
? entry.sourceAddr.split(":")[0]
: entry.sourceAddr;
const endpoint = ipToEndpoint.get(sourceIp);
let location: string | undefined;
if (endpoint) {
const endpointIp = endpoint.includes(":")
? endpoint.split(":")[0]
: endpoint;
location = await getCountryCodeForIp(endpointIp);
}
await logRequestAudit(
{
action: true,
reason: 108,
resourceId: entry.resourceId,
orgId
siteResourceId: entry.resourceId,
orgId,
location
},
{
path: entry.path,

149
server/routers/auditLogs/queryRequestAuditLog.ts
View File

@@ -1,8 +1,8 @@
import { logsDb, primaryLogsDb, requestAuditLog, resources, db, primaryDb } from "@server/db";
import { logsDb, primaryLogsDb, requestAuditLog, resources, siteResources, db, primaryDb } from "@server/db";
import { registry } from "@server/openApi";
import { NextFunction } from "express";
import { Request, Response } from "express";
import { eq, gt, lt, and, count, desc, inArray } from "drizzle-orm";
import { eq, gt, lt, and, count, desc, inArray, isNull, or } from "drizzle-orm";
import { OpenAPITags } from "@server/openApi";
import { z } from "zod";
import createHttpError from "http-errors";
@@ -92,7 +92,10 @@ function getWhere(data: Q) {
lt(requestAuditLog.timestamp, data.timeEnd),
eq(requestAuditLog.orgId, data.orgId),
data.resourceId
? eq(requestAuditLog.resourceId, data.resourceId)
? or(
eq(requestAuditLog.resourceId, data.resourceId),
eq(requestAuditLog.siteResourceId, data.resourceId)
)
: undefined,
data.actor ? eq(requestAuditLog.actor, data.actor) : undefined,
data.method ? eq(requestAuditLog.method, data.method) : undefined,
@@ -110,15 +113,16 @@ export function queryRequest(data: Q) {
return primaryLogsDb
.select({
id: requestAuditLog.id,
timestamp: requestAuditLog.timestamp,
orgId: requestAuditLog.orgId,
action: requestAuditLog.action,
reason: requestAuditLog.reason,
actorType: requestAuditLog.actorType,
actor: requestAuditLog.actor,
actorId: requestAuditLog.actorId,
resourceId: requestAuditLog.resourceId,
ip: requestAuditLog.ip,
timestamp: requestAuditLog.timestamp,
orgId: requestAuditLog.orgId,
action: requestAuditLog.action,
reason: requestAuditLog.reason,
actorType: requestAuditLog.actorType,
actor: requestAuditLog.actor,
actorId: requestAuditLog.actorId,
resourceId: requestAuditLog.resourceId,
siteResourceId: requestAuditLog.siteResourceId,
ip: requestAuditLog.ip,
location: requestAuditLog.location,
userAgent: requestAuditLog.userAgent,
metadata: requestAuditLog.metadata,
@@ -137,37 +141,73 @@ export function queryRequest(data: Q) {
}
async function enrichWithResourceDetails(logs: Awaited<ReturnType<typeof queryRequest>>) {
// If logs database is the same as main database, we can do a join
// Otherwise, we need to fetch resource details separately
const resourceIds = logs
.map(log => log.resourceId)
.filter((id): id is number => id !== null && id !== undefined);
if (resourceIds.length === 0) {
const siteResourceIds = logs
.filter(log => log.resourceId == null && log.siteResourceId != null)
.map(log => log.siteResourceId)
.filter((id): id is number => id !== null && id !== undefined);
if (resourceIds.length === 0 && siteResourceIds.length === 0) {
return logs.map(log => ({ ...log, resourceName: null, resourceNiceId: null }));
}
// Fetch resource details from main database
const resourceDetails = await primaryDb
.select({
resourceId: resources.resourceId,
name: resources.name,
niceId: resources.niceId
})
.from(resources)
.where(inArray(resources.resourceId, resourceIds));
const resourceMap = new Map<number, { name: string | null; niceId: string | null }>();
// Create a map for quick lookup
const resourceMap = new Map(
resourceDetails.map(r => [r.resourceId, { name: r.name, niceId: r.niceId }])
);
if (resourceIds.length > 0) {
const resourceDetails = await primaryDb
.select({
resourceId: resources.resourceId,
name: resources.name,
niceId: resources.niceId
})
.from(resources)
.where(inArray(resources.resourceId, resourceIds));
for (const r of resourceDetails) {
resourceMap.set(r.resourceId, { name: r.name, niceId: r.niceId });
}
}
const siteResourceMap = new Map<number, { name: string | null; niceId: string | null }>();
if (siteResourceIds.length > 0) {
const siteResourceDetails = await primaryDb
.select({
siteResourceId: siteResources.siteResourceId,
name: siteResources.name,
niceId: siteResources.niceId
})
.from(siteResources)
.where(inArray(siteResources.siteResourceId, siteResourceIds));
for (const r of siteResourceDetails) {
siteResourceMap.set(r.siteResourceId, { name: r.name, niceId: r.niceId });
}
}
// Enrich logs with resource details
return logs.map(log => ({
...log,
resourceName: log.resourceId ? resourceMap.get(log.resourceId)?.name ?? null : null,
resourceNiceId: log.resourceId ? resourceMap.get(log.resourceId)?.niceId ?? null : null
}));
return logs.map(log => {
if (log.resourceId != null) {
const details = resourceMap.get(log.resourceId);
return {
...log,
resourceName: details?.name ?? null,
resourceNiceId: details?.niceId ?? null
};
} else if (log.siteResourceId != null) {
const details = siteResourceMap.get(log.siteResourceId);
return {
...log,
resourceId: log.siteResourceId,
resourceName: details?.name ?? null,
resourceNiceId: details?.niceId ?? null
};
}
return { ...log, resourceName: null, resourceNiceId: null };
});
}
export function countRequestQuery(data: Q) {
@@ -211,7 +251,8 @@ async function queryUniqueFilterAttributes(
uniqueLocations,
uniqueHosts,
uniquePaths,
uniqueResources
uniqueResources,
uniqueSiteResources
] = await Promise.all([
primaryLogsDb
.selectDistinct({ actor: requestAuditLog.actor })
@@ -239,6 +280,13 @@ async function queryUniqueFilterAttributes(
})
.from(requestAuditLog)
.where(baseConditions)
.limit(DISTINCT_LIMIT + 1),
primaryLogsDb
.selectDistinct({
id: requestAuditLog.siteResourceId
})
.from(requestAuditLog)
.where(and(baseConditions, isNull(requestAuditLog.resourceId)))
.limit(DISTINCT_LIMIT + 1)
]);
@@ -259,6 +307,10 @@ async function queryUniqueFilterAttributes(
.map(row => row.id)
.filter((id): id is number => id !== null);
const siteResourceIds = uniqueSiteResources
.map(row => row.id)
.filter((id): id is number => id !== null);
let resourcesWithNames: Array<{ id: number; name: string | null }> = [];
if (resourceIds.length > 0) {
@@ -270,10 +322,31 @@ async function queryUniqueFilterAttributes(
.from(resources)
.where(inArray(resources.resourceId, resourceIds));
resourcesWithNames = resourceDetails.map(r => ({
id: r.resourceId,
name: r.name
}));
resourcesWithNames = [
...resourcesWithNames,
...resourceDetails.map(r => ({
id: r.resourceId,
name: r.name
}))
];
}
if (siteResourceIds.length > 0) {
const siteResourceDetails = await primaryDb
.select({
siteResourceId: siteResources.siteResourceId,
name: siteResources.name
})
.from(siteResources)
.where(inArray(siteResources.siteResourceId, siteResourceIds));
resourcesWithNames = [
...resourcesWithNames,
...siteResourceDetails.map(r => ({
id: r.siteResourceId,
name: r.name
}))
];
}
return {

1
server/routers/auditLogs/types.ts
View File

@@ -28,6 +28,7 @@ export type QueryRequestAuditLogResponse = {
actor: string | null;
actorId: string | null;
resourceId: number | null;
siteResourceId: number | null;
resourceNiceId: string | null;
resourceName: string | null;
ip: string | null;

3
server/routers/badger/logRequestAudit.ts
View File

@@ -39,6 +39,7 @@ const auditLogBuffer: Array<{
metadata: any;
action: boolean;
resourceId?: number;
siteResourceId?: number;
reason: number;
location?: string;
originalRequestURL: string;
@@ -187,6 +188,7 @@ export async function logRequestAudit(
action: boolean;
reason: number;
resourceId?: number;
siteResourceId?: number;
orgId?: string;
location?: string;
user?: { username: string; userId: string };
@@ -263,6 +265,7 @@ export async function logRequestAudit(
metadata: sanitizeString(metadata),
action: data.action,
resourceId: data.resourceId,
siteResourceId: data.siteResourceId,
reason: data.reason,
location: sanitizeString(data.location),
originalRequestURL: sanitizeString(body.originalRequestURL) ?? "",

6
src/app/[orgId]/settings/logs/request/page.tsx
View File

@@ -512,7 +512,11 @@ export default function GeneralPage() {
cell: ({ row }) => {
return (
<Link
href={`/${row.original.orgId}/settings/resources/proxy/${row.original.resourceNiceId}`}
href={
row.original.reason == 108 // for now the client will only have reason 108 so we know where to go
? `/${row.original.orgId}/settings/resources/client?query=${row.original.resourceNiceId}`
: `/${row.original.orgId}/settings/resources/proxy/${row.original.resourceNiceId}`
}
onClick={(e) => e.stopPropagation()}
>
<Button