mirror of
https://github.com/fosrl/pangolin.git
synced 2026-05-07 00:39:53 +00:00
Compare commits
8 Commits
redis
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
3a4b82acbc | ||
|
|
432dc81875 | ||
|
|
9b71c426c7 | ||
|
|
87e6c7ba36 | ||
|
|
9410a18404 | ||
|
|
23f4302186 | ||
|
|
fb4bda077b | ||
|
|
cf596d980f |
598
package-lock.json
generated
598
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
22
package.json
22
package.json
@@ -131,25 +131,25 @@
|
||||
"zod-validation-error": "5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@dotenvx/dotenvx": "1.54.1",
|
||||
"@dotenvx/dotenvx": "1.64.0",
|
||||
"@esbuild-plugins/tsconfig-paths": "0.1.2",
|
||||
"@react-email/preview-server": "5.2.10",
|
||||
"@tailwindcss/postcss": "4.2.2",
|
||||
"@tanstack/react-query-devtools": "5.91.3",
|
||||
"@tanstack/react-query-devtools": "5.100.9",
|
||||
"@types/better-sqlite3": "7.6.13",
|
||||
"@types/cookie-parser": "1.4.10",
|
||||
"@types/cors": "2.8.19",
|
||||
"@types/crypto-js": "4.2.2",
|
||||
"@types/d3": "7.4.3",
|
||||
"@types/express": "5.0.6",
|
||||
"@types/express-session": "1.18.2",
|
||||
"@types/express-session": "1.19.0",
|
||||
"@types/jmespath": "0.15.2",
|
||||
"@types/js-yaml": "4.0.9",
|
||||
"@types/jsonwebtoken": "9.0.10",
|
||||
"@types/node": "25.3.5",
|
||||
"@types/node": "25.6.0",
|
||||
"@types/nodemailer": "7.0.11",
|
||||
"@types/nprogress": "0.2.3",
|
||||
"@types/pg": "8.18.0",
|
||||
"@types/pg": "8.20.0",
|
||||
"@types/react": "19.2.14",
|
||||
"@types/react-dom": "19.2.3",
|
||||
"@types/semver": "7.7.1",
|
||||
@@ -160,10 +160,10 @@
|
||||
"@types/yargs": "17.0.35",
|
||||
"babel-plugin-react-compiler": "1.0.0",
|
||||
"drizzle-kit": "0.31.10",
|
||||
"esbuild": "0.27.4",
|
||||
"esbuild-node-externals": "1.20.1",
|
||||
"eslint": "10.0.3",
|
||||
"eslint-config-next": "16.1.7",
|
||||
"esbuild": "0.28.0",
|
||||
"esbuild-node-externals": "1.22.0",
|
||||
"eslint": "10.3.0",
|
||||
"eslint-config-next": "16.2.4",
|
||||
"postcss": "8.5.8",
|
||||
"prettier": "3.8.1",
|
||||
"react-email": "5.2.10",
|
||||
@@ -171,10 +171,10 @@
|
||||
"tsc-alias": "1.8.16",
|
||||
"tsx": "4.21.0",
|
||||
"typescript": "5.9.3",
|
||||
"typescript-eslint": "8.56.1"
|
||||
"typescript-eslint": "8.59.2"
|
||||
},
|
||||
"overrides": {
|
||||
"esbuild": "0.27.4",
|
||||
"esbuild": "0.28.0",
|
||||
"dompurify": "3.3.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
import { db, logsDb, statusHistory } from "@server/db";
|
||||
import { and, eq, gte, asc } from "drizzle-orm";
|
||||
import { regionalCache as cache } from "@server/private/lib/cache";
|
||||
import cache from "@server/lib/cache";
|
||||
|
||||
const STATUS_HISTORY_CACHE_TTL = 60; // seconds
|
||||
|
||||
@@ -63,7 +63,7 @@ export async function invalidateStatusHistoryCache(
|
||||
entityId: number
|
||||
): Promise<void> {
|
||||
const prefix = `statusHistory:${entityType}:${entityId}:`;
|
||||
const keys = await cache.keysWithPrefix(prefix);
|
||||
const keys = cache.keys().filter((k) => k.startsWith(prefix));
|
||||
if (keys.length > 0) {
|
||||
await cache.del(keys);
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
import NodeCache from "node-cache";
|
||||
import logger from "@server/logger";
|
||||
import { redisManager, regionalRedisManager } from "@server/private/lib/redis";
|
||||
import { redisManager } from "@server/private/lib/redis";
|
||||
|
||||
// Create local cache with maxKeys limit to prevent memory leaks
|
||||
// With ~10k requests/day and 5min TTL, 10k keys should be more than sufficient
|
||||
@@ -298,147 +298,3 @@ class AdaptiveCache {
|
||||
// Export singleton instance
|
||||
export const cache = new AdaptiveCache();
|
||||
export default cache;
|
||||
|
||||
/**
|
||||
* Regional adaptive cache backed by the in-cluster Redis instance.
|
||||
* Falls back to a local NodeCache when the regional Redis is unavailable.
|
||||
* Use this for data that is regional in nature (e.g. status history) so
|
||||
* reads are served from the same cluster the user is hitting.
|
||||
*/
|
||||
const regionalLocalCache = new NodeCache({
|
||||
stdTTL: 3600,
|
||||
checkperiod: 120,
|
||||
maxKeys: 10000
|
||||
});
|
||||
|
||||
class RegionalAdaptiveCache {
|
||||
private useRedis(): boolean {
|
||||
return (
|
||||
regionalRedisManager.isRedisEnabled() &&
|
||||
regionalRedisManager.getHealthStatus().isHealthy
|
||||
);
|
||||
}
|
||||
|
||||
async set(key: string, value: any, ttl?: number): Promise<boolean> {
|
||||
const effectiveTtl = ttl === 0 ? undefined : ttl;
|
||||
const redisTtl = ttl === 0 ? undefined : (ttl ?? 3600);
|
||||
|
||||
if (this.useRedis()) {
|
||||
try {
|
||||
const serialized = JSON.stringify(value);
|
||||
const success = await regionalRedisManager.set(
|
||||
key,
|
||||
serialized,
|
||||
redisTtl
|
||||
);
|
||||
if (success) {
|
||||
logger.debug(`[regional] Set key in Redis: ${key}`);
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[regional] Redis set error for key ${key}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const success = regionalLocalCache.set(key, value, effectiveTtl || 0);
|
||||
if (success) logger.debug(`[regional] Set key in local cache: ${key}`);
|
||||
return success;
|
||||
}
|
||||
|
||||
async get<T = any>(key: string): Promise<T | undefined> {
|
||||
if (this.useRedis()) {
|
||||
try {
|
||||
const value = await regionalRedisManager.get(key);
|
||||
if (value !== null) {
|
||||
logger.debug(`[regional] Cache hit in Redis: ${key}`);
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
logger.debug(`[regional] Cache miss in Redis: ${key}`);
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[regional] Redis get error for key ${key}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const value = regionalLocalCache.get<T>(key);
|
||||
if (value !== undefined) {
|
||||
logger.debug(`[regional] Cache hit in local cache: ${key}`);
|
||||
} else {
|
||||
logger.debug(`[regional] Cache miss in local cache: ${key}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async del(key: string | string[]): Promise<number> {
|
||||
const keys = Array.isArray(key) ? key : [key];
|
||||
let deletedCount = 0;
|
||||
|
||||
if (this.useRedis()) {
|
||||
try {
|
||||
for (const k of keys) {
|
||||
const success = await regionalRedisManager.del(k);
|
||||
if (success) {
|
||||
deletedCount++;
|
||||
logger.debug(`[regional] Deleted key from Redis: ${k}`);
|
||||
}
|
||||
}
|
||||
if (deletedCount === keys.length) return deletedCount;
|
||||
deletedCount = 0;
|
||||
} catch (error) {
|
||||
logger.error(`[regional] Redis del error:`, error);
|
||||
deletedCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
for (const k of keys) {
|
||||
const count = regionalLocalCache.del(k);
|
||||
if (count > 0) {
|
||||
deletedCount++;
|
||||
logger.debug(`[regional] Deleted key from local cache: ${k}`);
|
||||
}
|
||||
}
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
async has(key: string): Promise<boolean> {
|
||||
if (this.useRedis()) {
|
||||
try {
|
||||
const value = await regionalRedisManager.get(key);
|
||||
return value !== null;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[regional] Redis has error for key ${key}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
return regionalLocalCache.has(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns keys matching the given prefix from whichever backend is active.
|
||||
* Redis uses a KEYS scan; local cache filters in-memory keys.
|
||||
*/
|
||||
async keysWithPrefix(prefix: string): Promise<string[]> {
|
||||
if (this.useRedis()) {
|
||||
try {
|
||||
return await regionalRedisManager.keys(`${prefix}*`);
|
||||
} catch (error) {
|
||||
logger.error(`[regional] Redis keys error:`, error);
|
||||
}
|
||||
}
|
||||
return regionalLocalCache.keys().filter((k) => k.startsWith(prefix));
|
||||
}
|
||||
|
||||
getCurrentBackend(): "redis" | "local" {
|
||||
return this.useRedis() ? "redis" : "local";
|
||||
}
|
||||
}
|
||||
|
||||
export const regionalCache = new RegionalAdaptiveCache();
|
||||
|
||||
@@ -73,25 +73,6 @@ export const privateConfigSchema = z
|
||||
.object({
|
||||
rejectUnauthorized: z.boolean().optional().default(true)
|
||||
})
|
||||
.optional(),
|
||||
regional_redis: z
|
||||
.object({
|
||||
host: z.string(),
|
||||
port: portSchema,
|
||||
password: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform(getEnvOrYaml("REGIONAL_REDIS_PASSWORD")),
|
||||
db: z.int().nonnegative().optional().default(0),
|
||||
tls: z
|
||||
.object({
|
||||
rejectUnauthorized: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(true)
|
||||
})
|
||||
.optional()
|
||||
})
|
||||
.optional()
|
||||
})
|
||||
.optional(),
|
||||
|
||||
@@ -109,14 +109,14 @@ class RedisManager {
|
||||
password: redisConfig.password,
|
||||
db: redisConfig.db
|
||||
};
|
||||
|
||||
|
||||
// Enable TLS if configured (required for AWS ElastiCache in-transit encryption)
|
||||
if (redisConfig.tls) {
|
||||
opts.tls = {
|
||||
rejectUnauthorized: redisConfig.tls.rejectUnauthorized ?? true
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
return opts;
|
||||
}
|
||||
|
||||
@@ -135,14 +135,14 @@ class RedisManager {
|
||||
password: replica.password,
|
||||
db: replica.db || redisConfig.db
|
||||
};
|
||||
|
||||
|
||||
// Enable TLS if configured (required for AWS ElastiCache in-transit encryption)
|
||||
if (redisConfig.tls) {
|
||||
opts.tls = {
|
||||
rejectUnauthorized: redisConfig.tls.rejectUnauthorized ?? true
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
return opts;
|
||||
}
|
||||
|
||||
@@ -855,163 +855,3 @@ class RedisManager {
|
||||
export const redisManager = new RedisManager();
|
||||
export const redis = redisManager.getClient();
|
||||
export default redisManager;
|
||||
|
||||
/**
|
||||
* Lightweight Redis manager for the regional (in-cluster) Redis instance.
|
||||
* Connects only when `redis.regional_redis` is present in the private config
|
||||
* and `flags.enable_redis` is true. No pub/sub — designed for low-latency
|
||||
* caching of regionally-scoped data.
|
||||
*/
|
||||
class RegionalRedisManager {
|
||||
private writeClient: Redis | null = null;
|
||||
private readClient: Redis | null = null;
|
||||
private isEnabled: boolean = false;
|
||||
private isHealthy: boolean = false;
|
||||
private connectionTimeout: number = 5000;
|
||||
private commandTimeout: number = 5000;
|
||||
|
||||
constructor() {
|
||||
if (build === "oss") return;
|
||||
|
||||
const cfg = privateConfig.getRawPrivateConfig();
|
||||
if (!cfg.flags.enable_redis || !cfg.redis?.regional_redis) return;
|
||||
|
||||
this.isEnabled = true;
|
||||
this.initializeClients();
|
||||
}
|
||||
|
||||
private getConfig(): RedisOptions {
|
||||
const r = privateConfig.getRawPrivateConfig().redis!.regional_redis!;
|
||||
const opts: RedisOptions = {
|
||||
host: r.host,
|
||||
port: r.port,
|
||||
password: r.password,
|
||||
db: r.db
|
||||
};
|
||||
if (r.tls) {
|
||||
opts.tls = { rejectUnauthorized: r.tls.rejectUnauthorized ?? true };
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
private initializeClients(): void {
|
||||
const cfg = this.getConfig();
|
||||
const baseOpts = {
|
||||
...cfg,
|
||||
enableReadyCheck: false,
|
||||
maxRetriesPerRequest: 3,
|
||||
keepAlive: 10000,
|
||||
connectTimeout: this.connectionTimeout,
|
||||
commandTimeout: this.commandTimeout
|
||||
};
|
||||
|
||||
try {
|
||||
this.writeClient = new Redis(baseOpts);
|
||||
// redis-1 (replica) handles reads; fall back to primary if not resolvable
|
||||
this.readClient = new Redis({
|
||||
...baseOpts,
|
||||
host: cfg.host!.replace(/^(.*?)(\.\S+)$/, (_, h, rest) => {
|
||||
// Derive replica hostname from the headless service pattern:
|
||||
// redis.redis.svc.cluster.local -> redis-1.redis-headless.redis.svc.cluster.local
|
||||
// If it doesn't look like a k8s service, just use the same host
|
||||
return h + rest;
|
||||
})
|
||||
});
|
||||
|
||||
// For simplicity use same host for both; callers can always read from primary
|
||||
// The real replica routing is handled by the StatefulSet headless service
|
||||
this.readClient = this.writeClient;
|
||||
|
||||
this.writeClient.on("ready", () => {
|
||||
logger.info("Regional Redis client ready");
|
||||
this.isHealthy = true;
|
||||
});
|
||||
this.writeClient.on("error", (err) => {
|
||||
logger.error("Regional Redis client error:", err);
|
||||
this.isHealthy = false;
|
||||
});
|
||||
this.writeClient.on("reconnecting", () => {
|
||||
logger.info("Regional Redis client reconnecting...");
|
||||
this.isHealthy = false;
|
||||
});
|
||||
|
||||
logger.info("Regional Redis client initialized");
|
||||
} catch (error) {
|
||||
logger.error("Failed to initialize regional Redis client:", error);
|
||||
this.isEnabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
public isRedisEnabled(): boolean {
|
||||
return this.isEnabled && this.writeClient !== null && this.isHealthy;
|
||||
}
|
||||
|
||||
public getHealthStatus() {
|
||||
return { isEnabled: this.isEnabled, isHealthy: this.isHealthy };
|
||||
}
|
||||
|
||||
public async set(
|
||||
key: string,
|
||||
value: string,
|
||||
ttl?: number
|
||||
): Promise<boolean> {
|
||||
if (!this.isRedisEnabled() || !this.writeClient) return false;
|
||||
try {
|
||||
if (ttl) {
|
||||
await this.writeClient.setex(key, ttl, value);
|
||||
} else {
|
||||
await this.writeClient.set(key, value);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error("Regional Redis SET error:", error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async get(key: string): Promise<string | null> {
|
||||
if (!this.isRedisEnabled() || !this.readClient) return null;
|
||||
try {
|
||||
return await this.readClient.get(key);
|
||||
} catch (error) {
|
||||
logger.error("Regional Redis GET error:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async del(key: string): Promise<boolean> {
|
||||
if (!this.isRedisEnabled() || !this.writeClient) return false;
|
||||
try {
|
||||
await this.writeClient.del(key);
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error("Regional Redis DEL error:", error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async keys(pattern: string): Promise<string[]> {
|
||||
if (!this.isRedisEnabled() || !this.readClient) return [];
|
||||
try {
|
||||
return await this.readClient.keys(pattern);
|
||||
} catch (error) {
|
||||
logger.error("Regional Redis KEYS error:", error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public async disconnect(): Promise<void> {
|
||||
try {
|
||||
if (this.writeClient) {
|
||||
await this.writeClient.quit();
|
||||
this.writeClient = null;
|
||||
}
|
||||
this.readClient = null;
|
||||
logger.info("Regional Redis client disconnected");
|
||||
} catch (error) {
|
||||
logger.error("Error disconnecting regional Redis client:", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const regionalRedisManager = new RegionalRedisManager();
|
||||
|
||||
Reference in New Issue
Block a user