pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-02-20 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into user-compliance

Browse Source
This commit is contained in:
Owen
2025-10-27 10:37:53 -07:00
parent eeab92719a 77ba568c36
commit b59a6b82ef
105 changed files with 8762 additions and 776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/lib/billing/getOrgTierData.ts
View File

@@ -1,8 +1,8 @@
export async function getOrgTierData(
orgId: string
): Promise<{ tier: string | null; active: boolean }> {
let tier = null;
let active = false;
const tier = null;
const active = false;
return { tier, active };
}

12
server/lib/billing/usageService.ts
View File

@@ -1,5 +1,4 @@
import { eq, sql, and } from "drizzle-orm";
import NodeCache from "node-cache";
import { v4 as uuidv4 } from "uuid";
import { PutObjectCommand } from "@aws-sdk/client-s3";
import * as fs from "fs/promises";
@@ -20,6 +19,7 @@ import logger from "@server/logger";
import { sendToClient } from "#dynamic/routers/ws";
import { build } from "@server/build";
import { s3Client } from "@server/lib/s3";
import cache from "@server/lib/cache";
interface StripeEvent {
identifier?: string;
@@ -43,7 +43,6 @@ export function noop() {
}
export class UsageService {
private cache: NodeCache;
private bucketName: string | undefined;
private currentEventFile: string | null = null;
private currentFileStartTime: number = 0;
@@ -51,7 +50,6 @@ export class UsageService {
private uploadingFiles: Set<string> = new Set();
constructor() {
this.cache = new NodeCache({ stdTTL: 300 }); // 5 minute TTL
if (noop()) {
return;
}
@@ -399,7 +397,7 @@ export class UsageService {
featureId: FeatureId
): Promise<string | null> {
const cacheKey = `customer_${orgId}_${featureId}`;
const cached = this.cache.get<string>(cacheKey);
const cached = cache.get<string>(cacheKey);
if (cached) {
return cached;
@@ -422,7 +420,7 @@ export class UsageService {
const customerId = customer.customerId;
// Cache the result
this.cache.set(cacheKey, customerId);
cache.set(cacheKey, customerId, 300); // 5 minute TTL
return customerId;
} catch (error) {
@@ -700,10 +698,6 @@ export class UsageService {
await this.uploadFileToS3();
}
public clearCache(): void {
this.cache.flushAll();
}
/**
* Scan the events directory for files older than 1 minute and upload them if not empty.
*/

8
server/lib/blueprints/proxyResources.ts
View File

@@ -527,7 +527,7 @@ export async function updateProxyResources(
if (
existingRule.action !== getRuleAction(rule.action) ||
existingRule.match !== rule.match.toUpperCase() ||
existingRule.value !== rule.value
existingRule.value !== rule.value.toUpperCase()
) {
validateRule(rule);
await trx
@@ -535,7 +535,7 @@ export async function updateProxyResources(
.set({
action: getRuleAction(rule.action),
match: rule.match.toUpperCase(),
value: rule.value
value: rule.value.toUpperCase(),
})
.where(
eq(resourceRules.ruleId, existingRule.ruleId)
@@ -547,7 +547,7 @@ export async function updateProxyResources(
resourceId: existingResource.resourceId,
action: getRuleAction(rule.action),
match: rule.match.toUpperCase(),
value: rule.value,
value: rule.value.toUpperCase(),
priority: index + 1 // start priorities at 1
});
}
@@ -705,7 +705,7 @@ export async function updateProxyResources(
resourceId: newResource.resourceId,
action: getRuleAction(rule.action),
match: rule.match.toUpperCase(),
value: rule.value,
value: rule.value.toUpperCase(),
priority: index + 1 // start priorities at 1
});
}

36
server/lib/blueprints/types.ts
View File

@@ -275,24 +275,26 @@ export const ConfigSchema = z
}
)
.refine(
// Enforce proxy-port uniqueness within proxy-resources
// Enforce proxy-port uniqueness within proxy-resources per protocol
(config) => {
const proxyPortMap = new Map<number, string[]>();
const protocolPortMap = new Map<string, string[]>();
Object.entries(config["proxy-resources"]).forEach(
([resourceKey, resource]) => {
const proxyPort = resource["proxy-port"];
if (proxyPort !== undefined) {
if (!proxyPortMap.has(proxyPort)) {
proxyPortMap.set(proxyPort, []);
const protocol = resource.protocol;
if (proxyPort !== undefined && protocol !== undefined) {
const key = `${protocol}:${proxyPort}`;
if (!protocolPortMap.has(key)) {
protocolPortMap.set(key, []);
}
proxyPortMap.get(proxyPort)!.push(resourceKey);
protocolPortMap.get(key)!.push(resourceKey);
}
}
);
// Find duplicates
const duplicates = Array.from(proxyPortMap.entries()).filter(
const duplicates = Array.from(protocolPortMap.entries()).filter(
([_, resourceKeys]) => resourceKeys.length > 1
);
@@ -300,25 +302,29 @@ export const ConfigSchema = z
},
(config) => {
// Extract duplicates for error message
const proxyPortMap = new Map<number, string[]>();
const protocolPortMap = new Map<string, string[]>();
Object.entries(config["proxy-resources"]).forEach(
([resourceKey, resource]) => {
const proxyPort = resource["proxy-port"];
if (proxyPort !== undefined) {
if (!proxyPortMap.has(proxyPort)) {
proxyPortMap.set(proxyPort, []);
const protocol = resource.protocol;
if (proxyPort !== undefined && protocol !== undefined) {
const key = `${protocol}:${proxyPort}`;
if (!protocolPortMap.has(key)) {
protocolPortMap.set(key, []);
}
proxyPortMap.get(proxyPort)!.push(resourceKey);
protocolPortMap.get(key)!.push(resourceKey);
}
}
);
const duplicates = Array.from(proxyPortMap.entries())
const duplicates = Array.from(protocolPortMap.entries())
.filter(([_, resourceKeys]) => resourceKeys.length > 1)
.map(
([proxyPort, resourceKeys]) =>
`port ${proxyPort} used by proxy-resources: ${resourceKeys.join(", ")}`
([protocolPort, resourceKeys]) => {
const [protocol, port] = protocolPort.split(':');
return `${protocol.toUpperCase()} port ${port} used by proxy-resources: ${resourceKeys.join(", ")}`;
}
)
.join("; ");

5
server/lib/cache.ts Normal file
View File

@@ -0,0 +1,5 @@
import NodeCache from "node-cache";
export const cache = new NodeCache({ stdTTL: 3600, checkperiod: 120 });
export default cache;

62
server/lib/cleanupLogs.ts Normal file
View File

@@ -0,0 +1,62 @@
import { db, orgs } from "@server/db";
import { cleanUpOldLogs as cleanUpOldAccessLogs } from "@server/private/lib/logAccessAudit";
import { cleanUpOldLogs as cleanUpOldActionLogs } from "@server/private/middlewares/logActionAudit";
import { cleanUpOldLogs as cleanUpOldRequestLogs } from "@server/routers/badger/logRequestAudit";
import { gt, or } from "drizzle-orm";
export function initLogCleanupInterval() {
return setInterval(
async () => {
const orgsToClean = await db
.select({
orgId: orgs.orgId,
settingsLogRetentionDaysAction:
orgs.settingsLogRetentionDaysAction,
settingsLogRetentionDaysAccess:
orgs.settingsLogRetentionDaysAccess,
settingsLogRetentionDaysRequest:
orgs.settingsLogRetentionDaysRequest
})
.from(orgs)
.where(
or(
gt(orgs.settingsLogRetentionDaysAction, 0),
gt(orgs.settingsLogRetentionDaysAccess, 0),
gt(orgs.settingsLogRetentionDaysRequest, 0)
)
);
for (const org of orgsToClean) {
const {
orgId,
settingsLogRetentionDaysAction,
settingsLogRetentionDaysAccess,
settingsLogRetentionDaysRequest
} = org;
if (settingsLogRetentionDaysAction > 0) {
await cleanUpOldActionLogs(
orgId,
settingsLogRetentionDaysRequest
);
}
if (settingsLogRetentionDaysAccess > 0) {
await cleanUpOldAccessLogs(
orgId,
settingsLogRetentionDaysRequest
);
}
if (settingsLogRetentionDaysRequest > 0) {
await cleanUpOldRequestLogs(
orgId,
settingsLogRetentionDaysRequest
);
}
}
},
// 3 * 60 * 60 * 1000
60 * 1000 // for testing
); // every 3 hours
}

2
server/lib/geoip.ts
View File

@@ -6,7 +6,7 @@ export async function getCountryCodeForIp(
): Promise<string | undefined> {
try {
if (!maxmindLookup) {
logger.warn(
logger.debug(
"MaxMind DB path not configured, cannot perform GeoIP lookup"
);
return;

28
server/lib/serverIpService.ts Normal file
View File

@@ -0,0 +1,28 @@
import axios from "axios";
let serverIp: string | null = null;
const services = [
"https://ifconfig.io/ip",
"https://api.ipify.org",
"https://checkip.amazonaws.com"
];
export async function fetchServerIp() {
for (const url of services) {
try {
const response = await axios.get(url, { timeout: 5000 });
serverIp = response.data.trim();
console.log("Detected public IP:", serverIp);
return;
} catch (err: any) {
console.warn(`Failed to fetch server IP from ${url}: ${err.message || err.code}`);
}
}
console.error("All attempts to fetch server IP failed.");
}
export function getServerIp() {
return serverIp;
}

27
server/lib/traefik/TraefikConfigManager.ts
View File

@@ -309,10 +309,7 @@ export class TraefikConfigManager {
this.lastActiveDomains = new Set(domains);
}
if (
process.env.USE_PANGOLIN_DNS === "true" &&
build != "oss"
) {
if (process.env.USE_PANGOLIN_DNS === "true" && build != "oss") {
// Scan current local certificate state
this.lastLocalCertificateState =
await this.scanLocalCertificateState();
@@ -450,7 +447,8 @@ export class TraefikConfigManager {
currentExitNode,
config.getRawConfig().traefik.site_types,
build == "oss", // filter out the namespace domains in open source
build != "oss" // generate the login pages on the cloud and hybrid
build != "oss", // generate the login pages on the cloud and hybrid,
build == "saas" ? false : config.getRawConfig().traefik.allow_raw_resources // dont allow raw resources on saas otherwise use config
);
const domains = new Set<string>();
@@ -502,6 +500,25 @@ export class TraefikConfigManager {
};
}
// tcp:
// serversTransports:
// pp-transport-v1:
// proxyProtocol:
// version: 1
// pp-transport-v2:
// proxyProtocol:
// version: 2
if (build != "saas") {
// add the serversTransports section if not present
if (traefikConfig.tcp && !traefikConfig.tcp.serversTransports) {
traefikConfig.tcp.serversTransports = {
"pp-transport-v1": { proxyProtocol: { version: 1 } },
"pp-transport-v2": { proxyProtocol: { version: 2 } }
};
}
}
return { domains, traefikConfig };
} catch (error) {
// pull data out of the axios error to log

114
server/lib/traefik/getTraefikConfig.ts
View File

@@ -1,4 +1,4 @@
import { db, targetHealthCheck } from "@server/db";
import { db, targetHealthCheck, domains } from "@server/db";
import {
and,
eq,
@@ -23,7 +23,8 @@ export async function getTraefikConfig(
exitNodeId: number,
siteTypes: string[],
filterOutNamespaceDomains = false,
generateLoginPageRouters = false
generateLoginPageRouters = false,
allowRawResources = true
): Promise<any> {
// Define extended target type with site information
type TargetWithSite = Target & {
@@ -56,6 +57,8 @@ export async function getTraefikConfig(
setHostHeader: resources.setHostHeader,
enableProxy: resources.enableProxy,
headers: resources.headers,
proxyProtocol: resources.proxyProtocol,
proxyProtocolVersion: resources.proxyProtocolVersion,
// Target fields
targetId: targets.targetId,
targetEnabled: targets.enabled,
@@ -75,11 +78,14 @@ export async function getTraefikConfig(
siteType: sites.type,
siteOnline: sites.online,
subnet: sites.subnet,
exitNodeId: sites.exitNodeId
exitNodeId: sites.exitNodeId,
// Domain cert resolver fields
domainCertResolver: domains.certResolver
})
.from(sites)
.innerJoin(targets, eq(targets.siteId, sites.siteId))
.innerJoin(resources, eq(resources.resourceId, targets.resourceId))
.leftJoin(domains, eq(domains.domainId, resources.domainId))
.leftJoin(
targetHealthCheck,
eq(targetHealthCheck.targetId, targets.targetId)
@@ -101,7 +107,7 @@ export async function getTraefikConfig(
isNull(targetHealthCheck.hcHealth) // Include targets with no health check record
),
inArray(sites.type, siteTypes),
config.getRawConfig().traefik.allow_raw_resources
allowRawResources
? isNotNull(resources.http) // ignore the http check if allow_raw_resources is true
: eq(resources.http, true)
)
@@ -164,11 +170,15 @@ export async function getTraefikConfig(
enableProxy: row.enableProxy,
targets: [],
headers: row.headers,
proxyProtocol: row.proxyProtocol,
proxyProtocolVersion: row.proxyProtocolVersion ?? 1,
path: row.path, // the targets will all have the same path
pathMatchType: row.pathMatchType, // the targets will all have the same pathMatchType
rewritePath: row.rewritePath,
rewritePathType: row.rewritePathType,
priority: priority // may be null, we fallback later
priority: priority,
// Store domain cert resolver fields
domainCertResolver: row.domainCertResolver
});
}
@@ -247,30 +257,45 @@ export async function getTraefikConfig(
wildCard = resource.fullDomain;
}
const configDomain = config.getDomain(resource.domainId);
const globalDefaultResolver =
config.getRawConfig().traefik.cert_resolver;
const globalDefaultPreferWildcard =
config.getRawConfig().traefik.prefer_wildcard_cert;
let certResolver: string, preferWildcardCert: boolean;
if (!configDomain) {
certResolver = config.getRawConfig().traefik.cert_resolver;
preferWildcardCert =
config.getRawConfig().traefik.prefer_wildcard_cert;
} else {
certResolver = configDomain.cert_resolver;
preferWildcardCert = configDomain.prefer_wildcard_cert;
}
const domainCertResolver = resource.domainCertResolver;
const preferWildcardCert = resource.preferWildcardCert;
const tls = {
certResolver: certResolver,
...(preferWildcardCert
? {
domains: [
{
main: wildCard
}
]
}
: {})
};
let resolverName: string | undefined;
let preferWildcard: boolean | undefined;
// Handle both letsencrypt & custom cases
if (domainCertResolver) {
resolverName = domainCertResolver.trim();
} else {
resolverName = globalDefaultResolver;
}
if (
preferWildcardCert !== undefined &&
preferWildcardCert !== null
) {
preferWildcard = preferWildcardCert;
} else {
preferWildcard = globalDefaultPreferWildcard;
}
const tls = {
certResolver: resolverName,
...(preferWildcard
? {
domains: [
{
main: wildCard
}
]
}
: {})
};
const additionalMiddlewares =
config.getRawConfig().traefik.additional_middlewares || [];
@@ -509,14 +534,14 @@ export async function getTraefikConfig(
})(),
...(resource.stickySession
? {
sticky: {
cookie: {
name: "p_sticky", // TODO: make this configurable via config.yml like other cookies
secure: resource.ssl,
httpOnly: true
}
}
}
sticky: {
cookie: {
name: "p_sticky", // TODO: make this configurable via config.yml like other cookies
secure: resource.ssl,
httpOnly: true
}
}
}
: {})
}
};
@@ -615,15 +640,20 @@ export async function getTraefikConfig(
}
});
})(),
...(resource.proxyProtocol && protocol == "tcp"
? {
serversTransport: `pp-transport-v${resource.proxyProtocolVersion || 1}`
}
: {}),
...(resource.stickySession
? {
sticky: {
ipStrategy: {
depth: 0,
sourcePort: true
}
}
}
sticky: {
ipStrategy: {
depth: 0,
sourcePort: true
}
}
}
: {})
}
};