pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-02-21 20:36:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pull up downstream changes

Browse Source
This commit is contained in:
Owen
2025-07-13 21:57:24 -07:00
parent c679875273
commit 98a261e38c
108 changed files with 9799 additions and 2038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
server/db/names.ts
View File

@@ -59,7 +59,7 @@ export async function getUniqueExitNodeEndpointName(): Promise<string> {
export function generateName(): string {
return (
const name = (
names.descriptors[
Math.floor(Math.random() * names.descriptors.length)
] +
@@ -68,4 +68,7 @@ export function generateName(): string {
)
.toLowerCase()
.replace(/\s/g, "-");
// clean out any non-alphanumeric characters except for dashes
return name.replace(/[^a-z0-9-]/g, "");
}

23
server/db/pg/driver.ts
View File

@@ -1,4 +1,5 @@
import { drizzle as DrizzlePostgres } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
import { readConfigFile } from "@server/lib/readConfigFile";
import { withReplicas } from "drizzle-orm/pg-core";
@@ -20,19 +21,31 @@ function createDb() {
);
}
const primary = DrizzlePostgres(connectionString);
// Create connection pools instead of individual connections
const primaryPool = new Pool({
connectionString,
max: 20,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
});
const replicas = [];
if (!replicaConnections.length) {
replicas.push(primary);
replicas.push(DrizzlePostgres(primaryPool));
} else {
for (const conn of replicaConnections) {
const replica = DrizzlePostgres(conn.connection_string);
replicas.push(replica);
const replicaPool = new Pool({
connectionString: conn.connection_string,
max: 10,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
});
replicas.push(DrizzlePostgres(replicaPool));
}
}
return withReplicas(primary, replicas as any);
return withReplicas(DrizzlePostgres(primaryPool), replicas as any);
}
export const db = createDb();

2
server/db/pg/index.ts
View File

@@ -1,2 +1,2 @@
export * from "./driver";
export * from "./schema";
export * from "./schema";

26
server/db/pg/schema.ts
View File

@@ -14,6 +14,9 @@ export const domains = pgTable("domains", {
baseDomain: varchar("baseDomain").notNull(),
configManaged: boolean("configManaged").notNull().default(false),
type: varchar("type"), // "ns", "cname", "a"
verified: boolean("verified").notNull().default(false),
failed: boolean("failed").notNull().default(false),
tries: integer("tries").notNull().default(0)
});
export const orgs = pgTable("orgs", {
@@ -44,9 +47,9 @@ export const sites = pgTable("sites", {
}),
name: varchar("name").notNull(),
pubKey: varchar("pubKey"),
subnet: varchar("subnet").notNull(),
megabytesIn: real("bytesIn"),
megabytesOut: real("bytesOut"),
subnet: varchar("subnet"),
megabytesIn: real("bytesIn").default(0),
megabytesOut: real("bytesOut").default(0),
lastBandwidthUpdate: varchar("lastBandwidthUpdate"),
type: varchar("type").notNull(), // "newt" or "wireguard"
online: boolean("online").notNull().default(false),
@@ -282,18 +285,6 @@ export const userResources = pgTable("userResources", {
.references(() => resources.resourceId, { onDelete: "cascade" })
});
export const limitsTable = pgTable("limits", {
limitId: serial("limitId").primaryKey(),
orgId: varchar("orgId")
.references(() => orgs.orgId, {
onDelete: "cascade"
})
.notNull(),
name: varchar("name").notNull(),
value: bigint("value", { mode: "number" }).notNull(),
description: varchar("description")
});
export const userInvites = pgTable("userInvites", {
inviteId: varchar("inviteId").primaryKey(),
orgId: varchar("orgId")
@@ -520,7 +511,8 @@ export const clients = pgTable("clients", {
type: varchar("type").notNull(), // "olm"
online: boolean("online").notNull().default(false),
endpoint: varchar("endpoint"),
lastHolePunch: integer("lastHolePunch")
lastHolePunch: integer("lastHolePunch"),
maxConnections: integer("maxConnections")
});
export const clientSites = pgTable("clientSites", {
@@ -590,7 +582,6 @@ export type RoleSite = InferSelectModel<typeof roleSites>;
export type UserSite = InferSelectModel<typeof userSites>;
export type RoleResource = InferSelectModel<typeof roleResources>;
export type UserResource = InferSelectModel<typeof userResources>;
export type Limit = InferSelectModel<typeof limitsTable>;
export type UserInvite = InferSelectModel<typeof userInvites>;
export type UserOrg = InferSelectModel<typeof userOrgs>;
export type ResourceSession = InferSelectModel<typeof resourceSessions>;
@@ -613,3 +604,4 @@ export type Olm = InferSelectModel<typeof olms>;
export type OlmSession = InferSelectModel<typeof olmSessions>;
export type UserClient = InferSelectModel<typeof userClients>;
export type RoleClient = InferSelectModel<typeof roleClients>;
export type OrgDomains = InferSelectModel<typeof orgDomains>;

157
server/db/redis.ts
View File

@@ -3,30 +3,25 @@ import logger from "@server/logger";
import config from "@server/lib/config";
class RedisManager {
private static instance: RedisManager;
public client: Redis | null = null;
private subscriber: Redis | null = null;
private publisher: Redis | null = null;
private isEnabled: boolean = false;
private isHealthy: boolean = true;
private lastHealthCheck: number = 0;
private healthCheckInterval: number = 30000; // 30 seconds
private subscribers: Map<
string,
Set<(channel: string, message: string) => void>
> = new Map();
private constructor() {
constructor() {
this.isEnabled = config.getRawConfig().flags?.enable_redis || false;
if (this.isEnabled) {
this.initializeClients();
}
}
public static getInstance(): RedisManager {
if (!RedisManager.instance) {
RedisManager.instance = new RedisManager();
}
return RedisManager.instance;
}
private getRedisConfig(): RedisOptions {
const redisConfig = config.getRawConfig().redis!;
const opts: RedisOptions = {
@@ -34,38 +29,78 @@ class RedisManager {
port: redisConfig.port!,
password: redisConfig.password,
db: redisConfig.db,
tls: {
rejectUnauthorized:
redisConfig.tls?.reject_unauthorized || false
}
// tls: {
// rejectUnauthorized:
// redisConfig.tls?.reject_unauthorized || false
// }
};
return opts;
}
// Add reconnection logic in initializeClients
private initializeClients(): void {
const config = this.getRedisConfig();
try {
// Main client for general operations
this.client = new Redis(config);
this.client = new Redis({
...config,
enableReadyCheck: false,
maxRetriesPerRequest: 3,
keepAlive: 30000,
connectTimeout: 10000, // 10 seconds
commandTimeout: 5000, // 5 seconds
});
// Dedicated publisher client
this.publisher = new Redis(config);
this.publisher = new Redis({
...config,
enableReadyCheck: false,
maxRetriesPerRequest: 3,
keepAlive: 30000,
connectTimeout: 10000, // 10 seconds
commandTimeout: 5000, // 5 seconds
});
// Dedicated subscriber client
this.subscriber = new Redis(config);
this.subscriber = new Redis({
...config,
enableReadyCheck: false,
maxRetriesPerRequest: 3,
keepAlive: 30000,
connectTimeout: 10000, // 10 seconds
commandTimeout: 5000, // 5 seconds
});
// Set up error handlers
// Add reconnection handlers
this.client.on("error", (err) => {
logger.error("Redis client error:", err);
this.isHealthy = false;
});
this.client.on("reconnecting", () => {
logger.info("Redis client reconnecting...");
this.isHealthy = false;
});
this.client.on("ready", () => {
logger.info("Redis client ready");
this.isHealthy = true;
});
this.publisher.on("error", (err) => {
logger.error("Redis publisher error:", err);
this.isHealthy = false;
});
this.publisher.on("ready", () => {
logger.info("Redis publisher ready");
});
this.subscriber.on("error", (err) => {
logger.error("Redis subscriber error:", err);
this.isHealthy = false;
});
this.subscriber.on("ready", () => {
logger.info("Redis subscriber ready");
});
// Set up connection handlers
@@ -102,18 +137,65 @@ class RedisManager {
);
logger.info("Redis clients initialized successfully");
// Start periodic health monitoring
this.startHealthMonitoring();
} catch (error) {
logger.error("Failed to initialize Redis clients:", error);
this.isEnabled = false;
}
}
public isRedisEnabled(): boolean {
return this.isEnabled && this.client !== null;
private startHealthMonitoring(): void {
if (!this.isEnabled) return;
// Check health every 30 seconds
setInterval(async () => {
try {
await this.checkRedisHealth();
} catch (error) {
logger.error("Error during Redis health monitoring:", error);
}
}, this.healthCheckInterval);
}
public getClient(): Redis | null {
return this.client;
public isRedisEnabled(): boolean {
return this.isEnabled && this.client !== null && this.isHealthy;
}
private async checkRedisHealth(): Promise<boolean> {
const now = Date.now();
// Only check health every 30 seconds
if (now - this.lastHealthCheck < this.healthCheckInterval) {
return this.isHealthy;
}
this.lastHealthCheck = now;
if (!this.client) {
this.isHealthy = false;
return false;
}
try {
await Promise.race([
this.client.ping(),
new Promise((_, reject) =>
setTimeout(() => reject(new Error('Health check timeout')), 2000)
)
]);
this.isHealthy = true;
return true;
} catch (error) {
logger.error("Redis health check failed:", error);
this.isHealthy = false;
return false;
}
}
public getClient(): Redis {
return this.client!;
}
public async set(
@@ -247,11 +329,25 @@ class RedisManager {
public async publish(channel: string, message: string): Promise<boolean> {
if (!this.isRedisEnabled() || !this.publisher) return false;
// Quick health check before attempting to publish
const isHealthy = await this.checkRedisHealth();
if (!isHealthy) {
logger.warn("Skipping Redis publish due to unhealthy connection");
return false;
}
try {
await this.publisher.publish(channel, message);
// Add timeout to prevent hanging
await Promise.race([
this.publisher.publish(channel, message),
new Promise((_, reject) =>
setTimeout(() => reject(new Error('Redis publish timeout')), 3000)
)
]);
return true;
} catch (error) {
logger.error("Redis PUBLISH error:", error);
this.isHealthy = false; // Mark as unhealthy on error
return false;
}
}
@@ -267,13 +363,19 @@ class RedisManager {
if (!this.subscribers.has(channel)) {
this.subscribers.set(channel, new Set());
// Only subscribe to the channel if it's the first subscriber
await this.subscriber.subscribe(channel);
await Promise.race([
this.subscriber.subscribe(channel),
new Promise((_, reject) =>
setTimeout(() => reject(new Error('Redis subscribe timeout')), 5000)
)
]);
}
this.subscribers.get(channel)!.add(callback);
return true;
} catch (error) {
logger.error("Redis SUBSCRIBE error:", error);
this.isHealthy = false;
return false;
}
}
@@ -330,5 +432,6 @@ class RedisManager {
}
}
export const redisManager = RedisManager.getInstance();
export const redisManager = new RedisManager();
export const redis = redisManager.getClient();
export default redisManager;

22
server/db/sqlite/schema.ts
View File

@@ -7,7 +7,7 @@ export const domains = sqliteTable("domains", {
configManaged: integer("configManaged", { mode: "boolean" })
.notNull()
.default(false),
type: text("type"), // "ns", "cname", "a"
type: text("type") // "ns", "cname", "a"
});
export const orgs = sqliteTable("orgs", {
@@ -16,6 +16,15 @@ export const orgs = sqliteTable("orgs", {
subnet: text("subnet").notNull(),
});
export const userDomains = sqliteTable("userDomains", {
userId: text("userId")
.notNull()
.references(() => users.userId, { onDelete: "cascade" }),
domainId: text("domainId")
.notNull()
.references(() => domains.domainId, { onDelete: "cascade" })
});
export const orgDomains = sqliteTable("orgDomains", {
orgId: text("orgId")
.notNull()
@@ -38,9 +47,9 @@ export const sites = sqliteTable("sites", {
}),
name: text("name").notNull(),
pubKey: text("pubKey"),
subnet: text("subnet").notNull(),
megabytesIn: integer("bytesIn"),
megabytesOut: integer("bytesOut"),
subnet: text("subnet"),
megabytesIn: integer("bytesIn").default(0),
megabytesOut: integer("bytesOut").default(0),
lastBandwidthUpdate: text("lastBandwidthUpdate"),
type: text("type").notNull(), // "newt" or "wireguard"
online: integer("online", { mode: "boolean" }).notNull().default(false),
@@ -48,7 +57,7 @@ export const sites = sqliteTable("sites", {
// exit node stuff that is how to connect to the site when it has a wg server
address: text("address"), // this is the address of the wireguard interface in newt
endpoint: text("endpoint"), // this is how to reach gerbil externally - gets put into the wireguard config
publicKey: text("pubicKey"), // TODO: Fix typo in publicKey
publicKey: text("publicKey"), // TODO: Fix typo in publicKey
lastHolePunch: integer("lastHolePunch"),
listenPort: integer("listenPort"),
dockerSocketEnabled: integer("dockerSocketEnabled", { mode: "boolean" })
@@ -626,13 +635,14 @@ export type ResourceAccessToken = InferSelectModel<typeof resourceAccessToken>;
export type ResourceWhitelist = InferSelectModel<typeof resourceWhitelist>;
export type VersionMigration = InferSelectModel<typeof versionMigrations>;
export type ResourceRule = InferSelectModel<typeof resourceRules>;
export type Domain = InferSelectModel<typeof domains>;
export type Client = InferSelectModel<typeof clients>;
export type ClientSite = InferSelectModel<typeof clientSites>;
export type RoleClient = InferSelectModel<typeof roleClients>;
export type UserClient = InferSelectModel<typeof userClients>;
export type Domain = InferSelectModel<typeof domains>;
export type SupporterKey = InferSelectModel<typeof supporterKey>;
export type Idp = InferSelectModel<typeof idp>;
export type ApiKey = InferSelectModel<typeof apiKeys>;
export type ApiKeyAction = InferSelectModel<typeof apiKeyActions>;
export type ApiKeyOrg = InferSelectModel<typeof apiKeyOrg>;
export type OrgDomains = InferSelectModel<typeof orgDomains>;