pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-03-05 10:16:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat-blueprint-ui-on-dashboard

Browse Source
This commit is contained in:
Fred KISSIE
2025-10-29 03:31:51 +01:00
committed by GitHub
parent db49b599b5 772bda69f9
commit 02330a0756
169 changed files with 14164 additions and 1207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/lib/billing/getOrgTierData.ts
View File

@@ -1,8 +1,8 @@
export async function getOrgTierData(
orgId: string
): Promise<{ tier: string | null; active: boolean }> {
let tier = null;
let active = false;
const tier = null;
const active = false;
return { tier, active };
}

12
server/lib/billing/usageService.ts
View File

@@ -1,5 +1,4 @@
import { eq, sql, and } from "drizzle-orm";
import NodeCache from "node-cache";
import { v4 as uuidv4 } from "uuid";
import { PutObjectCommand } from "@aws-sdk/client-s3";
import * as fs from "fs/promises";
@@ -20,6 +19,7 @@ import logger from "@server/logger";
import { sendToClient } from "#dynamic/routers/ws";
import { build } from "@server/build";
import { s3Client } from "@server/lib/s3";
import cache from "@server/lib/cache";
interface StripeEvent {
identifier?: string;
@@ -43,7 +43,6 @@ export function noop() {
}
export class UsageService {
private cache: NodeCache;
private bucketName: string | undefined;
private currentEventFile: string | null = null;
private currentFileStartTime: number = 0;
@@ -51,7 +50,6 @@ export class UsageService {
private uploadingFiles: Set<string> = new Set();
constructor() {
this.cache = new NodeCache({ stdTTL: 300 }); // 5 minute TTL
if (noop()) {
return;
}
@@ -399,7 +397,7 @@ export class UsageService {
featureId: FeatureId
): Promise<string | null> {
const cacheKey = `customer_${orgId}_${featureId}`;
const cached = this.cache.get<string>(cacheKey);
const cached = cache.get<string>(cacheKey);
if (cached) {
return cached;
@@ -422,7 +420,7 @@ export class UsageService {
const customerId = customer.customerId;
// Cache the result
this.cache.set(cacheKey, customerId);
cache.set(cacheKey, customerId, 300); // 5 minute TTL
return customerId;
} catch (error) {
@@ -700,10 +698,6 @@ export class UsageService {
await this.uploadFileToS3();
}
public clearCache(): void {
this.cache.flushAll();
}
/**
* Scan the events directory for files older than 1 minute and upload them if not empty.
*/

8
server/lib/blueprints/proxyResources.ts
View File

@@ -527,7 +527,7 @@ export async function updateProxyResources(
if (
existingRule.action !== getRuleAction(rule.action) ||
existingRule.match !== rule.match.toUpperCase() ||
existingRule.value !== rule.value
existingRule.value !== rule.value.toUpperCase()
) {
validateRule(rule);
await trx
@@ -535,7 +535,7 @@ export async function updateProxyResources(
.set({
action: getRuleAction(rule.action),
match: rule.match.toUpperCase(),
value: rule.value
value: rule.value.toUpperCase(),
})
.where(
eq(resourceRules.ruleId, existingRule.ruleId)
@@ -547,7 +547,7 @@ export async function updateProxyResources(
resourceId: existingResource.resourceId,
action: getRuleAction(rule.action),
match: rule.match.toUpperCase(),
value: rule.value,
value: rule.value.toUpperCase(),
priority: index + 1 // start priorities at 1
});
}
@@ -705,7 +705,7 @@ export async function updateProxyResources(
resourceId: newResource.resourceId,
action: getRuleAction(rule.action),
match: rule.match.toUpperCase(),
value: rule.value,
value: rule.value.toUpperCase(),
priority: index + 1 // start priorities at 1
});
}

36
server/lib/blueprints/types.ts
View File

@@ -275,24 +275,26 @@ export const ConfigSchema = z
}
)
.refine(
// Enforce proxy-port uniqueness within proxy-resources
// Enforce proxy-port uniqueness within proxy-resources per protocol
(config) => {
const proxyPortMap = new Map<number, string[]>();
const protocolPortMap = new Map<string, string[]>();
Object.entries(config["proxy-resources"]).forEach(
([resourceKey, resource]) => {
const proxyPort = resource["proxy-port"];
if (proxyPort !== undefined) {
if (!proxyPortMap.has(proxyPort)) {
proxyPortMap.set(proxyPort, []);
const protocol = resource.protocol;
if (proxyPort !== undefined && protocol !== undefined) {
const key = `${protocol}:${proxyPort}`;
if (!protocolPortMap.has(key)) {
protocolPortMap.set(key, []);
}
proxyPortMap.get(proxyPort)!.push(resourceKey);
protocolPortMap.get(key)!.push(resourceKey);
}
}
);
// Find duplicates
const duplicates = Array.from(proxyPortMap.entries()).filter(
const duplicates = Array.from(protocolPortMap.entries()).filter(
([_, resourceKeys]) => resourceKeys.length > 1
);
@@ -300,25 +302,29 @@ export const ConfigSchema = z
},
(config) => {
// Extract duplicates for error message
const proxyPortMap = new Map<number, string[]>();
const protocolPortMap = new Map<string, string[]>();
Object.entries(config["proxy-resources"]).forEach(
([resourceKey, resource]) => {
const proxyPort = resource["proxy-port"];
if (proxyPort !== undefined) {
if (!proxyPortMap.has(proxyPort)) {
proxyPortMap.set(proxyPort, []);
const protocol = resource.protocol;
if (proxyPort !== undefined && protocol !== undefined) {
const key = `${protocol}:${proxyPort}`;
if (!protocolPortMap.has(key)) {
protocolPortMap.set(key, []);
}
proxyPortMap.get(proxyPort)!.push(resourceKey);
protocolPortMap.get(key)!.push(resourceKey);
}
}
);
const duplicates = Array.from(proxyPortMap.entries())
const duplicates = Array.from(protocolPortMap.entries())
.filter(([_, resourceKeys]) => resourceKeys.length > 1)
.map(
([proxyPort, resourceKeys]) =>
`port ${proxyPort} used by proxy-resources: ${resourceKeys.join(", ")}`
([protocolPort, resourceKeys]) => {
const [protocol, port] = protocolPort.split(':');
return `${protocol.toUpperCase()} port ${port} used by proxy-resources: ${resourceKeys.join(", ")}`;
}
)
.join("; ");

5
server/lib/cache.ts Normal file
View File

@@ -0,0 +1,5 @@
import NodeCache from "node-cache";
export const cache = new NodeCache({ stdTTL: 3600, checkperiod: 120 });
export default cache;

41
server/lib/checkOrgAccessPolicy.ts Normal file
View File

@@ -0,0 +1,41 @@
import { Org, ResourceSession, Session, User } from "@server/db";
export type CheckOrgAccessPolicyProps = {
orgId?: string;
org?: Org;
userId?: string;
user?: User;
sessionId?: string;
session?: Session;
};
export type CheckOrgAccessPolicyResult = {
allowed: boolean;
error?: string;
policies?: {
requiredTwoFactor?: boolean;
maxSessionLength?: {
compliant: boolean;
maxSessionLengthHours: number;
sessionAgeHours: number;
};
passwordAge?: {
compliant: boolean;
maxPasswordAgeDays: number;
passwordAgeDays: number;
};
};
};
export async function enforceResourceSessionLength(
resourceSession: ResourceSession,
org: Org
): Promise<{ valid: boolean; error?: string }> {
return { valid: true };
}
export async function checkOrgAccessPolicy(
props: CheckOrgAccessPolicyProps
): Promise<CheckOrgAccessPolicyResult> {
return { allowed: true };
}

62
server/lib/cleanupLogs.ts Normal file
View File

@@ -0,0 +1,62 @@
import { db, orgs } from "@server/db";
import { cleanUpOldLogs as cleanUpOldAccessLogs } from "#dynamic/lib/logAccessAudit";
import { cleanUpOldLogs as cleanUpOldActionLogs } from "#dynamic/middlewares/logActionAudit";
import { cleanUpOldLogs as cleanUpOldRequestLogs } from "@server/routers/badger/logRequestAudit";
import { gt, or } from "drizzle-orm";
export function initLogCleanupInterval() {
return setInterval(
async () => {
const orgsToClean = await db
.select({
orgId: orgs.orgId,
settingsLogRetentionDaysAction:
orgs.settingsLogRetentionDaysAction,
settingsLogRetentionDaysAccess:
orgs.settingsLogRetentionDaysAccess,
settingsLogRetentionDaysRequest:
orgs.settingsLogRetentionDaysRequest
})
.from(orgs)
.where(
or(
gt(orgs.settingsLogRetentionDaysAction, 0),
gt(orgs.settingsLogRetentionDaysAccess, 0),
gt(orgs.settingsLogRetentionDaysRequest, 0)
)
);
for (const org of orgsToClean) {
const {
orgId,
settingsLogRetentionDaysAction,
settingsLogRetentionDaysAccess,
settingsLogRetentionDaysRequest
} = org;
if (settingsLogRetentionDaysAction > 0) {
await cleanUpOldActionLogs(
orgId,
settingsLogRetentionDaysRequest
);
}
if (settingsLogRetentionDaysAccess > 0) {
await cleanUpOldAccessLogs(
orgId,
settingsLogRetentionDaysRequest
);
}
if (settingsLogRetentionDaysRequest > 0) {
await cleanUpOldRequestLogs(
orgId,
settingsLogRetentionDaysRequest
);
}
}
},
// 3 * 60 * 60 * 1000
60 * 1000 // for testing
); // every 3 hours
}

2
server/lib/consts.ts
View File

@@ -2,7 +2,7 @@ import path from "path";
import { fileURLToPath } from "url";
// This is a placeholder value replaced by the build process
export const APP_VERSION = "1.11.0";
export const APP_VERSION = "1.12.0";
export const __FILENAME = fileURLToPath(import.meta.url);
export const __DIRNAME = path.dirname(__FILENAME);

2
server/lib/geoip.ts
View File

@@ -6,7 +6,7 @@ export async function getCountryCodeForIp(
): Promise<string | undefined> {
try {
if (!maxmindLookup) {
logger.warn(
logger.debug(
"MaxMind DB path not configured, cannot perform GeoIP lookup"
);
return;

17
server/lib/logAccessAudit.ts Normal file
View File

@@ -0,0 +1,17 @@
export async function cleanUpOldLogs(orgId: string, retentionDays: number) {
return;
}
export async function logAccessAudit(data: {
action: boolean;
type: string;
orgId: string;
resourceId?: number;
user?: { username: string; userId: string };
apiKey?: { name: string | null; apiKeyId: string };
metadata?: any;
userAgent?: string;
requestIp?: string;
}) {
return;
}

2
server/lib/readConfigFile.ts
View File

@@ -50,7 +50,7 @@ export const configSchema = z
.string()
.nonempty("base_domain must not be empty")
.transform((url) => url.toLowerCase()),
cert_resolver: z.string().optional().default("letsencrypt"),
cert_resolver: z.string().optional(), // null falls back to traefik.cert_resolver
prefer_wildcard_cert: z.boolean().optional().default(false)
})
)

9
server/lib/resend.ts
View File

@@ -1,7 +1,8 @@
export enum AudienceIds {
General = "",
Subscribed = "",
Churned = ""
SignUps = "",
Subscribed = "",
Churned = "",
Newsletter = ""
}
let resend;
@@ -12,4 +13,4 @@ export async function moveEmailToAudience(
audienceId: AudienceIds
) {
return;
}
}

29
server/lib/serverIpService.ts Normal file
View File

@@ -0,0 +1,29 @@
import logger from "@server/logger";
import axios from "axios";
let serverIp: string | null = null;
const services = [
"https://checkip.amazonaws.com",
"https://ifconfig.io/ip",
"https://api.ipify.org",
];
export async function fetchServerIp() {
for (const url of services) {
try {
const response = await axios.get(url, { timeout: 5000 });
serverIp = response.data.trim();
logger.debug("Detected public IP: " + serverIp);
return;
} catch (err: any) {
console.warn(`Failed to fetch server IP from ${url}: ${err.message || err.code}`);
}
}
console.error("All attempts to fetch server IP failed.");
}
export function getServerIp() {
return serverIp;
}

65
server/lib/telemetry.ts
View File

@@ -200,10 +200,7 @@ class TelemetryClient {
event: "supporter_status",
properties: {
valid: stats.supporterStatus.valid,
tier: stats.supporterStatus.tier,
github_username: stats.supporterStatus.githubUsername
? this.anon(stats.supporterStatus.githubUsername)
: "None"
tier: stats.supporterStatus.tier
}
});
}
@@ -217,21 +214,6 @@ class TelemetryClient {
install_timestamp: hostMeta.createdAt
}
});
for (const email of stats.adminUsers) {
// There should only be on admin user, but just in case
if (email) {
this.client.capture({
distinctId: this.anon(email),
event: "admin_user",
properties: {
host_id: hostMeta.hostMetaId,
app_version: stats.appVersion,
hashed_email: this.anon(email)
}
});
}
}
}
private async collectAndSendAnalytics() {
@@ -262,19 +244,38 @@ class TelemetryClient {
num_clients: stats.numClients,
num_identity_providers: stats.numIdentityProviders,
num_sites_online: stats.numSitesOnline,
resources: stats.resources.map((r) => ({
name: this.anon(r.name),
sso_enabled: r.sso,
protocol: r.protocol,
http_enabled: r.http
})),
sites: stats.sites.map((s) => ({
site_name: this.anon(s.siteName),
megabytes_in: s.megabytesIn,
megabytes_out: s.megabytesOut,
type: s.type,
online: s.online
})),
num_resources_sso_enabled: stats.resources.filter(
(r) => r.sso
).length,
num_resources_non_http: stats.resources.filter(
(r) => !r.http
).length,
num_newt_sites: stats.sites.filter((s) => s.type === "newt")
.length,
num_local_sites: stats.sites.filter(
(s) => s.type === "local"
).length,
num_wg_sites: stats.sites.filter(
(s) => s.type === "wireguard"
).length,
avg_megabytes_in:
stats.sites.length > 0
? Math.round(
stats.sites.reduce(
(sum, s) => sum + (s.megabytesIn ?? 0),
0
) / stats.sites.length
)
: 0,
avg_megabytes_out:
stats.sites.length > 0
? Math.round(
stats.sites.reduce(
(sum, s) => sum + (s.megabytesOut ?? 0),
0
) / stats.sites.length
)
: 0,
num_api_keys: stats.numApiKeys,
num_custom_roles: stats.numCustomRoles
}

27
server/lib/traefik/TraefikConfigManager.ts
View File

@@ -309,10 +309,7 @@ export class TraefikConfigManager {
this.lastActiveDomains = new Set(domains);
}
if (
process.env.USE_PANGOLIN_DNS === "true" &&
build != "oss"
) {
if (process.env.USE_PANGOLIN_DNS === "true" && build != "oss") {
// Scan current local certificate state
this.lastLocalCertificateState =
await this.scanLocalCertificateState();
@@ -450,7 +447,8 @@ export class TraefikConfigManager {
currentExitNode,
config.getRawConfig().traefik.site_types,
build == "oss", // filter out the namespace domains in open source
build != "oss" // generate the login pages on the cloud and hybrid
build != "oss", // generate the login pages on the cloud and hybrid,
build == "saas" ? false : config.getRawConfig().traefik.allow_raw_resources // dont allow raw resources on saas otherwise use config
);
const domains = new Set<string>();
@@ -502,6 +500,25 @@ export class TraefikConfigManager {
};
}
// tcp:
// serversTransports:
// pp-transport-v1:
// proxyProtocol:
// version: 1
// pp-transport-v2:
// proxyProtocol:
// version: 2
if (build != "saas") {
// add the serversTransports section if not present
if (traefikConfig.tcp && !traefikConfig.tcp.serversTransports) {
traefikConfig.tcp.serversTransports = {
"pp-transport-v1": { proxyProtocol: { version: 1 } },
"pp-transport-v2": { proxyProtocol: { version: 2 } }
};
}
}
return { domains, traefikConfig };
} catch (error) {
// pull data out of the axios error to log

123
server/lib/traefik/getTraefikConfig.ts
View File

@@ -1,4 +1,4 @@
import { db, targetHealthCheck } from "@server/db";
import { db, targetHealthCheck, domains } from "@server/db";
import {
and,
eq,
@@ -23,7 +23,8 @@ export async function getTraefikConfig(
exitNodeId: number,
siteTypes: string[],
filterOutNamespaceDomains = false,
generateLoginPageRouters = false
generateLoginPageRouters = false,
allowRawResources = true
): Promise<any> {
// Define extended target type with site information
type TargetWithSite = Target & {
@@ -56,6 +57,8 @@ export async function getTraefikConfig(
setHostHeader: resources.setHostHeader,
enableProxy: resources.enableProxy,
headers: resources.headers,
proxyProtocol: resources.proxyProtocol,
proxyProtocolVersion: resources.proxyProtocolVersion,
// Target fields
targetId: targets.targetId,
targetEnabled: targets.enabled,
@@ -75,11 +78,14 @@ export async function getTraefikConfig(
siteType: sites.type,
siteOnline: sites.online,
subnet: sites.subnet,
exitNodeId: sites.exitNodeId
exitNodeId: sites.exitNodeId,
// Domain cert resolver fields
domainCertResolver: domains.certResolver
})
.from(sites)
.innerJoin(targets, eq(targets.siteId, sites.siteId))
.innerJoin(resources, eq(resources.resourceId, targets.resourceId))
.leftJoin(domains, eq(domains.domainId, resources.domainId))
.leftJoin(
targetHealthCheck,
eq(targetHealthCheck.targetId, targets.targetId)
@@ -88,13 +94,20 @@ export async function getTraefikConfig(
and(
eq(targets.enabled, true),
eq(resources.enabled, true),
eq(sites.exitNodeId, exitNodeId),
or(
eq(sites.exitNodeId, exitNodeId),
and(
isNull(sites.exitNodeId),
sql`(${siteTypes.includes("local") ? 1 : 0} = 1)`, // only allow local sites if "local" is in siteTypes
eq(sites.type, "local")
)
),
or(
ne(targetHealthCheck.hcHealth, "unhealthy"), // Exclude unhealthy targets
isNull(targetHealthCheck.hcHealth) // Include targets with no health check record
),
inArray(sites.type, siteTypes),
config.getRawConfig().traefik.allow_raw_resources
allowRawResources
? isNotNull(resources.http) // ignore the http check if allow_raw_resources is true
: eq(resources.http, true)
)
@@ -157,11 +170,15 @@ export async function getTraefikConfig(
enableProxy: row.enableProxy,
targets: [],
headers: row.headers,
proxyProtocol: row.proxyProtocol,
proxyProtocolVersion: row.proxyProtocolVersion ?? 1,
path: row.path, // the targets will all have the same path
pathMatchType: row.pathMatchType, // the targets will all have the same pathMatchType
rewritePath: row.rewritePath,
rewritePathType: row.rewritePathType,
priority: priority // may be null, we fallback later
priority: priority,
// Store domain cert resolver fields
domainCertResolver: row.domainCertResolver
});
}
@@ -240,30 +257,45 @@ export async function getTraefikConfig(
wildCard = resource.fullDomain;
}
const configDomain = config.getDomain(resource.domainId);
const globalDefaultResolver =
config.getRawConfig().traefik.cert_resolver;
const globalDefaultPreferWildcard =
config.getRawConfig().traefik.prefer_wildcard_cert;
let certResolver: string, preferWildcardCert: boolean;
if (!configDomain) {
certResolver = config.getRawConfig().traefik.cert_resolver;
preferWildcardCert =
config.getRawConfig().traefik.prefer_wildcard_cert;
} else {
certResolver = configDomain.cert_resolver;
preferWildcardCert = configDomain.prefer_wildcard_cert;
}
const domainCertResolver = resource.domainCertResolver;
const preferWildcardCert = resource.preferWildcardCert;
const tls = {
certResolver: certResolver,
...(preferWildcardCert
? {
domains: [
{
main: wildCard
}
]
}
: {})
};
let resolverName: string | undefined;
let preferWildcard: boolean | undefined;
// Handle both letsencrypt & custom cases
if (domainCertResolver) {
resolverName = domainCertResolver.trim();
} else {
resolverName = globalDefaultResolver;
}
if (
preferWildcardCert !== undefined &&
preferWildcardCert !== null
) {
preferWildcard = preferWildcardCert;
} else {
preferWildcard = globalDefaultPreferWildcard;
}
const tls = {
certResolver: resolverName,
...(preferWildcard
? {
domains: [
{
main: wildCard
}
]
}
: {})
};
const additionalMiddlewares =
config.getRawConfig().traefik.additional_middlewares || [];
@@ -502,14 +534,14 @@ export async function getTraefikConfig(
})(),
...(resource.stickySession
? {
sticky: {
cookie: {
name: "p_sticky", // TODO: make this configurable via config.yml like other cookies
secure: resource.ssl,
httpOnly: true
}
}
}
sticky: {
cookie: {
name: "p_sticky", // TODO: make this configurable via config.yml like other cookies
secure: resource.ssl,
httpOnly: true
}
}
}
: {})
}
};
@@ -608,15 +640,20 @@ export async function getTraefikConfig(
}
});
})(),
...(resource.proxyProtocol && protocol == "tcp"
? {
serversTransport: `pp-transport-v${resource.proxyProtocolVersion || 1}`
}
: {}),
...(resource.stickySession
? {
sticky: {
ipStrategy: {
depth: 0,
sourcePort: true
}
}
}
sticky: {
ipStrategy: {
depth: 0,
sourcePort: true
}
}
}
: {})
}
};