mirror of
https://github.com/fosrl/pangolin.git
synced 2026-07-06 05:49:53 +00:00
Compare commits
30 Commits
dependabot
...
crowdin_de
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3db28e5140 | ||
|
|
2e95257e09 | ||
|
|
dcd6d6486d | ||
|
|
df145ac14c | ||
|
|
00de9ed49f | ||
|
|
3b6ce77972 | ||
|
|
5589e4da1d | ||
|
|
b662a4c06f | ||
|
|
b4d0354740 | ||
|
|
949a0221d8 | ||
|
|
c4b56b2b29 | ||
|
|
da7676de14 | ||
|
|
d2e7438692 | ||
|
|
b0f196b4ec | ||
|
|
ef9d15fded | ||
|
|
04a4a1899d | ||
|
|
f9c3d0647d | ||
|
|
6bac384b19 | ||
|
|
bf4426f1f9 | ||
|
|
7bf419d058 | ||
|
|
e85ebe717c | ||
|
|
2ec527f5d7 | ||
|
|
d334e6c3a5 | ||
|
|
755281abc3 | ||
|
|
7b05ad6b1d | ||
|
|
edbd1a743e | ||
|
|
fa5397095d | ||
|
|
3f914c4eab | ||
|
|
18b1460615 | ||
|
|
267ee63a60 |
10
.github/workflows/cicd.yml
vendored
10
.github/workflows/cicd.yml
vendored
@@ -77,7 +77,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: docker.io
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: docker.io
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
@@ -204,7 +204,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: docker.io
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
@@ -264,7 +264,7 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: 1.25
|
||||
|
||||
@@ -407,7 +407,7 @@ jobs:
|
||||
shell: bash
|
||||
|
||||
- name: Login to GitHub Container Registry (for cosign)
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -41,7 +41,7 @@ services:
|
||||
- 80:80 # Port for traefik because of the network_mode
|
||||
|
||||
traefik:
|
||||
image: traefik:v3.7
|
||||
image: traefik:v3.6
|
||||
container_name: traefik
|
||||
restart: unless-stopped
|
||||
network_mode: service:gerbil # Ports appear on the gerbil service
|
||||
|
||||
@@ -50,7 +50,7 @@ services:
|
||||
- 80:80{{end}}
|
||||
|
||||
traefik:
|
||||
image: docker.io/traefik:v3.7
|
||||
image: docker.io/traefik:v3.6
|
||||
container_name: traefik
|
||||
restart: unless-stopped
|
||||
{{if .InstallGerbil}}network_mode: service:gerbil # Ports appear on the gerbil service{{end}}{{if not .InstallGerbil}}
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Базово удостоверяване чрез заглавие",
|
||||
"policyAuthHeaderAuthDescription": "Валидирайте собствено HTTP заглавие и стойност при всяка заявка",
|
||||
"policyAuthHeaderAuthSummary": "Конфигурирано заглавие",
|
||||
"policyAuthHeaderName": "Име на заглавието",
|
||||
"policyAuthHeaderValue": "Очаквана стойност",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Задайте парола",
|
||||
"policyAuthSetPincode": "Задайте ПИН код",
|
||||
"policyAuthSetEmailWhitelist": "Задайте списък с имейли",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Домейни",
|
||||
"billingOrganizations": "Организации",
|
||||
"billingRemoteExitNodes": "Дистанционни възли",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Няма конфигуриран лимит",
|
||||
"billingEstimatedPeriod": "Очакван период на фактуриране",
|
||||
"billingIncludedUsage": "Включено използване",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Колко потребители можете да използвате",
|
||||
"billingDomainInfo": "Колко домейни можете да използвате",
|
||||
"billingRemoteExitNodesInfo": "Колко дистанционни възли можете да използвате",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Лицензионни ключове",
|
||||
"billingLicenseKeysDescription": "Управлявайте вашите абонаменти за лицензионни ключове",
|
||||
"billingLicenseSubscription": "Абонамент за лиценз",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Základní Ověření Záhlaví",
|
||||
"policyAuthHeaderAuthDescription": "Ověřit vlastní HTTP hlavičku názvu a hodnoty při každém požadavku",
|
||||
"policyAuthHeaderAuthSummary": "Nastaveno hlavička",
|
||||
"policyAuthHeaderName": "Název hlavičky",
|
||||
"policyAuthHeaderValue": "Očekávaná hodnota",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Nastavit přístupový kód",
|
||||
"policyAuthSetPincode": "Nastavit PIN kód",
|
||||
"policyAuthSetEmailWhitelist": "Nastavit e-mailový whitelist",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domény",
|
||||
"billingOrganizations": "Tělo",
|
||||
"billingRemoteExitNodes": "Vzdálené uzly",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Žádný limit nenastaven",
|
||||
"billingEstimatedPeriod": "Odhadované období fakturace",
|
||||
"billingIncludedUsage": "Zahrnuto využití",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Kolik uživatelů můžete použít",
|
||||
"billingDomainInfo": "Kolik domén můžete použít",
|
||||
"billingRemoteExitNodesInfo": "Kolik vzdálených uzlů můžete použít",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Licenční klíče",
|
||||
"billingLicenseKeysDescription": "Spravovat předplatné licenčního klíče",
|
||||
"billingLicenseSubscription": "Předplatné licence",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Grunnleggende Header Autentificering",
|
||||
"policyAuthHeaderAuthDescription": "Bekræft et tilpasset HTTP-headernavn og værdi ved hver forespørgsel",
|
||||
"policyAuthHeaderAuthSummary": "Header konfigureret",
|
||||
"policyAuthHeaderName": "Headernavn",
|
||||
"policyAuthHeaderValue": "Forventet værdi",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Angiv adgangskode",
|
||||
"policyAuthSetPincode": "Sett PIN-kode",
|
||||
"policyAuthSetEmailWhitelist": "Angiv e-mail hviteliste",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domæner",
|
||||
"billingOrganizations": "Orger",
|
||||
"billingRemoteExitNodes": "Eksterne noder",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Ingen grænse konfigureret",
|
||||
"billingEstimatedPeriod": "Estimert faktureringsperiode",
|
||||
"billingIncludedUsage": "Inklusive Brug",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Hvor mange brugere du kan bruge",
|
||||
"billingDomainInfo": "Hvor mange domæner du kan bruge",
|
||||
"billingRemoteExitNodesInfo": "Hvor mange fjernnoder du kan bruge",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Licensnøgler",
|
||||
"billingLicenseKeysDescription": "Administrer dine licensnøgleabonnementer",
|
||||
"billingLicenseSubscription": "Licens abonnement",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Grundlegende Header-Authentifizierung",
|
||||
"policyAuthHeaderAuthDescription": "Überprüfen Sie einen benutzerdefinierten HTTP-Headernamen und -wert bei jeder Anfrage",
|
||||
"policyAuthHeaderAuthSummary": "Header konfiguriert",
|
||||
"policyAuthHeaderName": "Header-Name",
|
||||
"policyAuthHeaderValue": "Erwarteter Wert",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Passcode setzen",
|
||||
"policyAuthSetPincode": "PIN-Code festlegen",
|
||||
"policyAuthSetEmailWhitelist": "E-Mail-Whitelist festlegen",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domänen",
|
||||
"billingOrganizations": "Orden",
|
||||
"billingRemoteExitNodes": "Entfernte Knoten",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Kein Limit konfiguriert",
|
||||
"billingEstimatedPeriod": "Geschätzter Abrechnungszeitraum",
|
||||
"billingIncludedUsage": "Inklusive Nutzung",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Wie viele Benutzer Sie verwenden können",
|
||||
"billingDomainInfo": "Wie viele Domains Sie verwenden können",
|
||||
"billingRemoteExitNodesInfo": "Wie viele entfernte Knoten Sie verwenden können",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Lizenzschlüssel",
|
||||
"billingLicenseKeysDescription": "Verwalten Sie Ihre Lizenzschlüssel Abonnements",
|
||||
"billingLicenseSubscription": "Lizenzabonnement",
|
||||
|
||||
@@ -1915,9 +1915,6 @@
|
||||
"billingDomains": "Domains",
|
||||
"billingOrganizations": "Orgs",
|
||||
"billingRemoteExitNodes": "Remote Nodes",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "No limit configured",
|
||||
"billingEstimatedPeriod": "Estimated Billing Period",
|
||||
"billingIncludedUsage": "Included Usage",
|
||||
@@ -1946,9 +1943,6 @@
|
||||
"billingUsersInfo": "How many users you can use",
|
||||
"billingDomainInfo": "How many domains you can use",
|
||||
"billingRemoteExitNodesInfo": "How many remote nodes you can use",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "License Keys",
|
||||
"billingLicenseKeysDescription": "Manage your license key subscriptions",
|
||||
"billingLicenseSubscription": "License Subscription",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Autenticación Básica del Encabezado",
|
||||
"policyAuthHeaderAuthDescription": "Valida un nombre y valor de encabezado HTTP personalizado en cada petición",
|
||||
"policyAuthHeaderAuthSummary": "Encabezado configurado",
|
||||
"policyAuthHeaderName": "Nombre del encabezado",
|
||||
"policyAuthHeaderValue": "Valor esperado",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Establecer Código de Acceso",
|
||||
"policyAuthSetPincode": "Establecer Código PIN",
|
||||
"policyAuthSetEmailWhitelist": "Establecer Lista Blanca de Correo",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Dominios",
|
||||
"billingOrganizations": "Orgánico",
|
||||
"billingRemoteExitNodes": "Nodos remotos",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "No se ha configurado ningún límite",
|
||||
"billingEstimatedPeriod": "Período de facturación estimado",
|
||||
"billingIncludedUsage": "Uso incluido",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Cuántos usuarios puedes usar",
|
||||
"billingDomainInfo": "Cuántos dominios puedes usar",
|
||||
"billingRemoteExitNodesInfo": "Cuántos nodos remotos puedes usar",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Claves de licencia",
|
||||
"billingLicenseKeysDescription": "Administrar las suscripciones de su clave de licencia",
|
||||
"billingLicenseSubscription": "Suscripción de licencia",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Authentification de l'en-tête de base",
|
||||
"policyAuthHeaderAuthDescription": "Validez un nom et une valeur d'en-tête HTTP personnalisé à chaque requête",
|
||||
"policyAuthHeaderAuthSummary": "En-tête configuré",
|
||||
"policyAuthHeaderName": "Nom de l'en-tête",
|
||||
"policyAuthHeaderValue": "Valeur attendue",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Définir le code confidentiel",
|
||||
"policyAuthSetPincode": "Définir le code PIN",
|
||||
"policyAuthSetEmailWhitelist": "Définir la liste blanche des e-mails",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domaines",
|
||||
"billingOrganizations": "Organes",
|
||||
"billingRemoteExitNodes": "Nœuds distants",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Aucune limite configurée",
|
||||
"billingEstimatedPeriod": "Période de facturation estimée",
|
||||
"billingIncludedUsage": "Utilisation incluse",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Combien d'utilisateurs vous pouvez utiliser",
|
||||
"billingDomainInfo": "Combien de domaines vous pouvez utiliser",
|
||||
"billingRemoteExitNodesInfo": "Combien de nœuds distants vous pouvez utiliser",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Clés de licence",
|
||||
"billingLicenseKeysDescription": "Gérer vos abonnements à la clé de licence",
|
||||
"billingLicenseSubscription": "Abonnement à la licence",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Autenticazione Header Base",
|
||||
"policyAuthHeaderAuthDescription": "Convalida un nome e un valore di intestazione HTTP personalizzato su ogni richiesta",
|
||||
"policyAuthHeaderAuthSummary": "Intestazione configurata",
|
||||
"policyAuthHeaderName": "Nome dell'intestazione",
|
||||
"policyAuthHeaderValue": "Valore atteso",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Imposta Codice di Accesso",
|
||||
"policyAuthSetPincode": "Imposta Codice PIN",
|
||||
"policyAuthSetEmailWhitelist": "Imposta Lista Autorizzazioni Email",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domini",
|
||||
"billingOrganizations": "Organi",
|
||||
"billingRemoteExitNodes": "Nodi Remoti",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Nessun limite configurato",
|
||||
"billingEstimatedPeriod": "Periodo di Fatturazione Stimato",
|
||||
"billingIncludedUsage": "Utilizzo Incluso",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Quanti utenti puoi usare",
|
||||
"billingDomainInfo": "Quanti domini puoi usare",
|
||||
"billingRemoteExitNodesInfo": "Quanti nodi remoti puoi usare",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Chiavi di Licenza",
|
||||
"billingLicenseKeysDescription": "Gestisci le sottoscrizioni alla chiave di licenza",
|
||||
"billingLicenseSubscription": "Abbonamento Licenza",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "기본 헤더 인증",
|
||||
"policyAuthHeaderAuthDescription": "각 요청에서 맞춤 HTTP 헤더 이름 및 값을 검증",
|
||||
"policyAuthHeaderAuthSummary": "헤더 구성됨",
|
||||
"policyAuthHeaderName": "헤더 이름",
|
||||
"policyAuthHeaderValue": "예상 값",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "패스코드 설정",
|
||||
"policyAuthSetPincode": "PIN 코드 설정",
|
||||
"policyAuthSetEmailWhitelist": "이메일 화이트리스트 설정",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "도메인",
|
||||
"billingOrganizations": "조직",
|
||||
"billingRemoteExitNodes": "원격 노드",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "구성된 한도가 없습니다.",
|
||||
"billingEstimatedPeriod": "예상 청구 기간",
|
||||
"billingIncludedUsage": "포함 사용량",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "사용할 수 있는 사용자 수",
|
||||
"billingDomainInfo": "사용할 수 있는 도메인 수",
|
||||
"billingRemoteExitNodesInfo": "사용할 수 있는 원격 노드 수",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "라이센스 키",
|
||||
"billingLicenseKeysDescription": "라이센스 키 구독을 관리하세요",
|
||||
"billingLicenseSubscription": "라이센스 구독",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Grunnleggende Header Autentisering",
|
||||
"policyAuthHeaderAuthDescription": "Bekreft et tilpasset HTTP-headernavn og verdi ved hver forespørsel",
|
||||
"policyAuthHeaderAuthSummary": "Header konfigurert",
|
||||
"policyAuthHeaderName": "Headernavn",
|
||||
"policyAuthHeaderValue": "Forventet verdi",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Angi passordkode",
|
||||
"policyAuthSetPincode": "Sett PIN-kode",
|
||||
"policyAuthSetEmailWhitelist": "Angi e-post hviteliste",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domener",
|
||||
"billingOrganizations": "Orger",
|
||||
"billingRemoteExitNodes": "Eksterne Noder",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Ingen grense konfigurert",
|
||||
"billingEstimatedPeriod": "Estimert faktureringsperiode",
|
||||
"billingIncludedUsage": "Inkludert Bruk",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Hvor mange brukere du kan bruke",
|
||||
"billingDomainInfo": "Hvor mange domener du kan bruke",
|
||||
"billingRemoteExitNodesInfo": "Hvor mange fjernnoder du kan bruke",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Lisensnøkler",
|
||||
"billingLicenseKeysDescription": "Administrer dine lisensnøkkelabonnementer",
|
||||
"billingLicenseSubscription": "Lisens abonnement",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Basic Header Authenticatie",
|
||||
"policyAuthHeaderAuthDescription": "Valideer een aangepaste HTTP-headernaam en waarde bij elk verzoek",
|
||||
"policyAuthHeaderAuthSummary": "Header geconfigureerd",
|
||||
"policyAuthHeaderName": "Header naam",
|
||||
"policyAuthHeaderValue": "Verwachte waarde",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Stel toegangscode in",
|
||||
"policyAuthSetPincode": "Stel Pincode in",
|
||||
"policyAuthSetEmailWhitelist": "Stel E-mail Whitelist in",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domeinen",
|
||||
"billingOrganizations": "Ordenen",
|
||||
"billingRemoteExitNodes": "Externe knooppunten",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Geen limiet ingesteld",
|
||||
"billingEstimatedPeriod": "Geschatte Facturatie Periode",
|
||||
"billingIncludedUsage": "Opgenomen Gebruik",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Hoeveel gebruikers je kan gebruiken",
|
||||
"billingDomainInfo": "Hoeveel domeinen je kunt gebruiken",
|
||||
"billingRemoteExitNodesInfo": "Hoeveel externe nodes je kunt gebruiken",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Licentie Sleutels",
|
||||
"billingLicenseKeysDescription": "Beheer uw licentiesleutelabonnementen",
|
||||
"billingLicenseSubscription": "Licentie abonnement",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Podstawowe Uwierzytelnianie Nagłówka",
|
||||
"policyAuthHeaderAuthDescription": "Walidacja niestandardowej nazwy i wartości nagłówka HTTP przy każdym żądaniu",
|
||||
"policyAuthHeaderAuthSummary": "Skonfigurowany nagłówek",
|
||||
"policyAuthHeaderName": "Nazwa nagłówka",
|
||||
"policyAuthHeaderValue": "Oczekiwana wartość",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Ustaw hasło dostępu",
|
||||
"policyAuthSetPincode": "Ustaw kod PIN",
|
||||
"policyAuthSetEmailWhitelist": "Ustaw białą listę e-mail",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domeny",
|
||||
"billingOrganizations": "O masie całkowitej pojazdu przekraczającej 5 ton, ale nieprzekraczającej 5 ton",
|
||||
"billingRemoteExitNodes": "Zdalne węzły",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Nie skonfigurowano limitu",
|
||||
"billingEstimatedPeriod": "Szacowany Okres Rozliczeniowy",
|
||||
"billingIncludedUsage": "Zawarte użycie",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Ile użytkowników możesz użyć",
|
||||
"billingDomainInfo": "Ile domen możesz użyć",
|
||||
"billingRemoteExitNodesInfo": "Ile zdalnych węzłów możesz użyć",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Klucze licencyjne",
|
||||
"billingLicenseKeysDescription": "Zarządzaj subskrypcjami kluczy licencyjnych",
|
||||
"billingLicenseSubscription": "Subskrypcja licencji",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Autenticação de Cabeçalho Básico",
|
||||
"policyAuthHeaderAuthDescription": "Valide um nome e valor de cabeçalho HTTP personalizado em cada solicitação",
|
||||
"policyAuthHeaderAuthSummary": "Cabeçalho configurado",
|
||||
"policyAuthHeaderName": "Nome do Cabeçalho",
|
||||
"policyAuthHeaderValue": "Valor esperado",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Definir Código de Acesso",
|
||||
"policyAuthSetPincode": "Definir Código PIN",
|
||||
"policyAuthSetEmailWhitelist": "Definir Lista de E-mails Permitidos",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Domínios",
|
||||
"billingOrganizations": "Órgãos",
|
||||
"billingRemoteExitNodes": "Nós remotos",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Nenhum limite configurado",
|
||||
"billingEstimatedPeriod": "Período Estimado de Cobrança",
|
||||
"billingIncludedUsage": "Uso Incluído",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Quantos usuários você pode usar",
|
||||
"billingDomainInfo": "Quantos domínios você pode usar",
|
||||
"billingRemoteExitNodesInfo": "Quantos nós remotos você pode usar",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Chaves de Licença",
|
||||
"billingLicenseKeysDescription": "Gerenciar suas subscrições de chave de licença",
|
||||
"billingLicenseSubscription": "Assinatura de Licença",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Базовая аутентификация заголовка",
|
||||
"policyAuthHeaderAuthDescription": "Проверка пользовательского имени и значения HTTP-заголовка для каждого запроса",
|
||||
"policyAuthHeaderAuthSummary": "Заголовок настроен",
|
||||
"policyAuthHeaderName": "Имя заголовка",
|
||||
"policyAuthHeaderValue": "Ожидаемое значение",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Установить пароль",
|
||||
"policyAuthSetPincode": "Установить ПИН-код",
|
||||
"policyAuthSetEmailWhitelist": "Установить белый список email",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Домены",
|
||||
"billingOrganizations": "Орги",
|
||||
"billingRemoteExitNodes": "Удаленные узлы",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Лимит не установлен",
|
||||
"billingEstimatedPeriod": "Предполагаемый период выставления счетов",
|
||||
"billingIncludedUsage": "Включенное использование",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Сколько пользователей вы можете использовать",
|
||||
"billingDomainInfo": "Сколько доменов вы можете использовать",
|
||||
"billingRemoteExitNodesInfo": "Сколько удаленных узлов вы можете использовать",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Лицензионные ключи",
|
||||
"billingLicenseKeysDescription": "Управление подписками на лицензионные ключи",
|
||||
"billingLicenseSubscription": "Лицензионное соглашение",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "Temel Başlık Kimlik Doğrulama",
|
||||
"policyAuthHeaderAuthDescription": "Her istekte özel bir HTTP başlık adını ve değerini doğrulayın",
|
||||
"policyAuthHeaderAuthSummary": "Başlık yapılandırıldı",
|
||||
"policyAuthHeaderName": "Başlık adı",
|
||||
"policyAuthHeaderValue": "Beklenen değer",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "Şifreyi Ayarla",
|
||||
"policyAuthSetPincode": "PIN Kodunu Ayarla",
|
||||
"policyAuthSetEmailWhitelist": "E-posta Beyaz Listesini Ayarla",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "Alan Adları",
|
||||
"billingOrganizations": "Organizasyonlar",
|
||||
"billingRemoteExitNodes": "Uzak Düğümler",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "Hiçbir limit yapılandırılmadı",
|
||||
"billingEstimatedPeriod": "Tahmini Fatura Dönemi",
|
||||
"billingIncludedUsage": "Dahil Kullanım",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "Kaç tane kullanıcı kullanabileceğiniz",
|
||||
"billingDomainInfo": "Kaç tane alan adı kullanabileceğiniz",
|
||||
"billingRemoteExitNodesInfo": "Kaç tane uzaktan düğüm kullanabileceğiniz",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "Lisans Anahtarları",
|
||||
"billingLicenseKeysDescription": "Lisans anahtarı aboneliklerinizi yönetin",
|
||||
"billingLicenseSubscription": "Lisans Aboneliği",
|
||||
|
||||
@@ -864,8 +864,8 @@
|
||||
"policyAuthHeaderAuthTitle": "基础标题认证",
|
||||
"policyAuthHeaderAuthDescription": "在每次请求上验证自定义 HTTP 标头名称和值",
|
||||
"policyAuthHeaderAuthSummary": "标头已配置",
|
||||
"policyAuthHeaderName": "标头名称",
|
||||
"policyAuthHeaderValue": "预期值",
|
||||
"policyAuthHeaderName": "Username",
|
||||
"policyAuthHeaderValue": "Password",
|
||||
"policyAuthSetPasscode": "设定密码",
|
||||
"policyAuthSetPincode": "设置 PIN 码",
|
||||
"policyAuthSetEmailWhitelist": "设置电子邮件白名单",
|
||||
@@ -1915,6 +1915,9 @@
|
||||
"billingDomains": "域",
|
||||
"billingOrganizations": "球队",
|
||||
"billingRemoteExitNodes": "远程节点",
|
||||
"billingPublicResources": "Public Resources",
|
||||
"billingPrivateResources": "Private Resources",
|
||||
"billingMachineClients": "Machine Clients",
|
||||
"billingNoLimitConfigured": "未配置限制",
|
||||
"billingEstimatedPeriod": "估计结算周期",
|
||||
"billingIncludedUsage": "包含的使用量",
|
||||
@@ -1943,6 +1946,9 @@
|
||||
"billingUsersInfo": "您可以使用多少用户",
|
||||
"billingDomainInfo": "您可以使用多少域",
|
||||
"billingRemoteExitNodesInfo": "您可以使用多少远程节点",
|
||||
"billingPublicResourcesInfo": "How many public resources you can use",
|
||||
"billingPrivateResourcesInfo": "How many private resources you can use",
|
||||
"billingMachineClientsInfo": "How many machine clients you can use",
|
||||
"billingLicenseKeys": "许可证密钥",
|
||||
"billingLicenseKeysDescription": "管理您的许可证密钥订阅",
|
||||
"billingLicenseSubscription": "许可订阅",
|
||||
|
||||
@@ -27,12 +27,6 @@ export async function getFeatureDisplayName(
|
||||
return "Remote Exit Nodes";
|
||||
case LimitId.ORGANIZATIONS:
|
||||
return "Organizations";
|
||||
case LimitId.PUBLIC_RESOURCES:
|
||||
return "Public Resources";
|
||||
case LimitId.PRIVATE_RESOURCES:
|
||||
return "Private Resources";
|
||||
case LimitId.MACHINE_CLIENTS:
|
||||
return "Machine Clients";
|
||||
case LimitId.TIER1:
|
||||
return "Home Lab";
|
||||
default:
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
import logger from "@server/logger";
|
||||
|
||||
const MAX_RETRIES = 5;
|
||||
const BASE_DELAY_MS = 50;
|
||||
|
||||
/**
|
||||
* Detect transient errors that are safe to retry (connection drops, deadlocks,
|
||||
* serialization failures). PostgreSQL deadlocks (40P01) are always safe to
|
||||
* retry: the database guarantees exactly one winner per deadlock pair, so the
|
||||
* loser just needs to try again.
|
||||
*/
|
||||
export function isTransientError(error: any): boolean {
|
||||
if (!error) return false;
|
||||
|
||||
const message = (error.message || "").toLowerCase();
|
||||
const causeMessage = (error.cause?.message || "").toLowerCase();
|
||||
const code = error.code || error.cause?.code || "";
|
||||
|
||||
// Connection timeout / terminated
|
||||
if (
|
||||
message.includes("connection timeout") ||
|
||||
message.includes("connection terminated") ||
|
||||
message.includes("timeout exceeded when trying to connect") ||
|
||||
causeMessage.includes("connection terminated unexpectedly") ||
|
||||
causeMessage.includes("connection timeout")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// PostgreSQL deadlock detected - always safe to retry (one winner guaranteed)
|
||||
if (code === "40P01" || message.includes("deadlock")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// PostgreSQL serialization failure
|
||||
if (code === "40001") {
|
||||
return true;
|
||||
}
|
||||
|
||||
// ECONNRESET, ECONNREFUSED, EPIPE, ETIMEDOUT
|
||||
if (
|
||||
code === "ECONNRESET" ||
|
||||
code === "ECONNREFUSED" ||
|
||||
code === "EPIPE" ||
|
||||
code === "ETIMEDOUT"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple retry wrapper with exponential backoff for transient errors
|
||||
* (deadlocks, connection timeouts, unexpected disconnects).
|
||||
*/
|
||||
export async function withRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
context: string,
|
||||
maxRetries: number = MAX_RETRIES,
|
||||
baseDelayMs: number = BASE_DELAY_MS
|
||||
): Promise<T> {
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error: any) {
|
||||
if (isTransientError(error) && attempt < maxRetries) {
|
||||
attempt++;
|
||||
const baseDelay = Math.pow(2, attempt - 1) * baseDelayMs;
|
||||
const jitter = Math.random() * baseDelay;
|
||||
const delay = baseDelay + jitter;
|
||||
logger.warn(
|
||||
`Transient DB error in ${context}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`,
|
||||
{ code: error?.code ?? error?.cause?.code }
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +1,30 @@
|
||||
import { db, exitNodes, Transaction } from "@server/db";
|
||||
import { db, exitNodes } from "@server/db";
|
||||
import config from "@server/lib/config";
|
||||
import { findNextAvailableCidr } from "@server/lib/ip";
|
||||
import { lockManager } from "#dynamic/lib/lock";
|
||||
|
||||
/**
|
||||
* Reserves the next available exit node subnet.
|
||||
*
|
||||
* Exit node subnets must never overlap with one another - regardless of
|
||||
* which org(s) they belong to - since HA exit nodes can end up routing for
|
||||
* the same org. This acquires a lock that the caller MUST release (via the
|
||||
* returned `release`) only after the chosen address has been durably
|
||||
* persisted (e.g. after the enclosing transaction commits), otherwise
|
||||
* concurrent callers can race and pick the same subnet.
|
||||
*/
|
||||
export async function getNextAvailableSubnet(
|
||||
trx: Transaction | typeof db = db
|
||||
): Promise<{ value: string; release: () => Promise<void> }> {
|
||||
const lockKey = "exit-node-subnet-allocation";
|
||||
const acquired = await lockManager.acquireLockWithRetry(lockKey, 6000);
|
||||
if (!acquired) {
|
||||
throw new Error(`Failed to acquire lock: ${lockKey}`);
|
||||
export async function getNextAvailableSubnet(): Promise<string> {
|
||||
// Get all existing subnets from routes table
|
||||
const existingAddresses = await db
|
||||
.select({
|
||||
address: exitNodes.address
|
||||
})
|
||||
.from(exitNodes);
|
||||
|
||||
const addresses = existingAddresses.map((a) => a.address);
|
||||
let subnet = findNextAvailableCidr(
|
||||
addresses,
|
||||
config.getRawConfig().gerbil.block_size,
|
||||
config.getRawConfig().gerbil.subnet_group
|
||||
);
|
||||
if (!subnet) {
|
||||
throw new Error("No available subnets remaining in space");
|
||||
}
|
||||
const release = () => lockManager.releaseLock(lockKey, acquired);
|
||||
|
||||
try {
|
||||
// Get all existing subnets from routes table
|
||||
const existingAddresses = await trx
|
||||
.select({
|
||||
address: exitNodes.address
|
||||
})
|
||||
.from(exitNodes);
|
||||
|
||||
const addresses = existingAddresses.map((a) => a.address);
|
||||
let subnet = findNextAvailableCidr(
|
||||
addresses,
|
||||
config.getRawConfig().gerbil.block_size,
|
||||
config.getRawConfig().gerbil.subnet_group
|
||||
);
|
||||
if (!subnet) {
|
||||
throw new Error("No available subnets remaining in space");
|
||||
}
|
||||
|
||||
// replace the last octet with 1
|
||||
subnet =
|
||||
subnet.split(".").slice(0, 3).join(".") +
|
||||
".1" +
|
||||
"/" +
|
||||
subnet.split("/")[1];
|
||||
return { value: subnet, release };
|
||||
} catch (e) {
|
||||
await release();
|
||||
throw e;
|
||||
}
|
||||
// replace the last octet with 1
|
||||
subnet =
|
||||
subnet.split(".").slice(0, 3).join(".") +
|
||||
".1" +
|
||||
"/" +
|
||||
subnet.split("/")[1];
|
||||
return subnet;
|
||||
}
|
||||
|
||||
@@ -45,7 +45,6 @@ import {
|
||||
} from "@server/routers/client/targets";
|
||||
import { lockManager } from "#dynamic/lib/lock";
|
||||
import { rebuildQueue } from "#dynamic/lib/rebuildQueue";
|
||||
import { withRetry, isTransientError } from "@server/lib/dbRetry";
|
||||
import {
|
||||
checkOrgRebuildRateLimit,
|
||||
decrementOrgRebuildCount,
|
||||
@@ -286,20 +285,10 @@ export async function rebuildClientAssociationsFromSiteResource(
|
||||
) {
|
||||
await incrementOrgRebuildCount(siteResource.orgId);
|
||||
try {
|
||||
// The whole locked rebuild is idempotent (it diffs full expected vs.
|
||||
// actual state each time), so on a transient DB error it's safe to
|
||||
// retry the entire thing rather than just the failed query.
|
||||
return await withRetry(
|
||||
() =>
|
||||
lockManager.withLock(
|
||||
`rebuild-client-associations:site-resource:${siteResource.siteResourceId}`,
|
||||
() =>
|
||||
rebuildClientAssociationsFromSiteResourceImpl(
|
||||
siteResource
|
||||
),
|
||||
REBUILD_ASSOCIATIONS_LOCK_TTL_MS
|
||||
),
|
||||
`rebuildClientAssociationsFromSiteResource:${siteResource.siteResourceId}`
|
||||
return await lockManager.withLock(
|
||||
`rebuild-client-associations:site-resource:${siteResource.siteResourceId}`,
|
||||
() => rebuildClientAssociationsFromSiteResourceImpl(siteResource),
|
||||
REBUILD_ASSOCIATIONS_LOCK_TTL_MS
|
||||
);
|
||||
} catch (err: any) {
|
||||
if (
|
||||
@@ -315,17 +304,6 @@ export async function rebuildClientAssociationsFromSiteResource(
|
||||
});
|
||||
return { mergedAllClients: [] };
|
||||
}
|
||||
if (isTransientError(err)) {
|
||||
logger.warn(
|
||||
`rebuildClientAssociations: transient DB error rebuilding site resource ${siteResource.siteResourceId} persisted after retries, queuing for deferred processing:`,
|
||||
err
|
||||
);
|
||||
await rebuildQueue.enqueue({
|
||||
type: "site-resource",
|
||||
id: siteResource.siteResourceId
|
||||
});
|
||||
return { mergedAllClients: [] };
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
await decrementOrgRebuildCount(siteResource.orgId);
|
||||
@@ -485,7 +463,6 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
|
||||
await trx
|
||||
.insert(clientSiteResourcesAssociationsCache)
|
||||
.values(clientSiteResourcesToInsert)
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteResourceId=${siteResource.siteResourceId} inserted clientSiteResource associations`
|
||||
@@ -533,148 +510,121 @@ async function rebuildClientAssociationsFromSiteResourceImpl(
|
||||
for (const site of sitesToProcess) {
|
||||
const siteId = site.siteId;
|
||||
|
||||
try {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] processing siteId=${siteId} for siteResourceId=${siteResource.siteResourceId}`
|
||||
);
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] processing siteId=${siteId} for siteResourceId=${siteResource.siteResourceId}`
|
||||
);
|
||||
|
||||
const existingClientSites = await trx
|
||||
.select({
|
||||
clientId: clientSitesAssociationsCache.clientId
|
||||
})
|
||||
.from(clientSitesAssociationsCache)
|
||||
.where(eq(clientSitesAssociationsCache.siteId, siteId));
|
||||
const existingClientSites = await trx
|
||||
.select({
|
||||
clientId: clientSitesAssociationsCache.clientId
|
||||
})
|
||||
.from(clientSitesAssociationsCache)
|
||||
.where(eq(clientSitesAssociationsCache.siteId, siteId));
|
||||
|
||||
const existingClientSiteIds = existingClientSites.map(
|
||||
(row) => row.clientId
|
||||
);
|
||||
const existingClientSiteIds = existingClientSites.map(
|
||||
(row) => row.clientId
|
||||
);
|
||||
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} existingClientSiteIds=[${existingClientSiteIds.join(", ")}]`
|
||||
);
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} existingClientSiteIds=[${existingClientSiteIds.join(", ")}]`
|
||||
);
|
||||
|
||||
// Get full client details for existing clients (needed for sending delete messages)
|
||||
const existingClients =
|
||||
existingClientSiteIds.length > 0
|
||||
? await trx
|
||||
.select({
|
||||
clientId: clients.clientId,
|
||||
pubKey: clients.pubKey,
|
||||
subnet: clients.subnet
|
||||
})
|
||||
.from(clients)
|
||||
.where(
|
||||
inArray(clients.clientId, existingClientSiteIds)
|
||||
)
|
||||
: [];
|
||||
|
||||
const otherResourceClientIds =
|
||||
clientsFromOtherResourcesBySite.get(siteId) ??
|
||||
new Set<number>();
|
||||
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} otherResourceClientIds=[${[...otherResourceClientIds].join(", ")}] mergedAllClientIds=[${mergedAllClientIds.join(", ")}]`
|
||||
);
|
||||
|
||||
// Expected clients from this resource are site-scoped: if this site is
|
||||
// no longer attached to the resource, the expected set is empty.
|
||||
const expectedClientIdsForSite = currentSiteIdSet.has(siteId)
|
||||
? mergedAllClientIds
|
||||
// Get full client details for existing clients (needed for sending delete messages)
|
||||
const existingClients =
|
||||
existingClientSiteIds.length > 0
|
||||
? await trx
|
||||
.select({
|
||||
clientId: clients.clientId,
|
||||
pubKey: clients.pubKey,
|
||||
subnet: clients.subnet
|
||||
})
|
||||
.from(clients)
|
||||
.where(inArray(clients.clientId, existingClientSiteIds))
|
||||
: [];
|
||||
|
||||
// Note: we deliberately do NOT exclude clients covered by another
|
||||
// site resource here (unlike clientSitesToRemove below). Doing so
|
||||
// previously caused a permanent gap: if resource A saw resource B's
|
||||
// cache row and skipped adding (assuming B would maintain it), and
|
||||
// B's own rebuild made the same assumption about A, the site-level
|
||||
// row could end up never inserted by anyone even though both
|
||||
// resources' client associations were otherwise correct.
|
||||
// onConflictDoNothing makes a redundant insert harmless, so there's
|
||||
// no correctness reason to skip here.
|
||||
const clientSitesToAdd = expectedClientIdsForSite.filter(
|
||||
(clientId) => !existingClientSiteIds.includes(clientId)
|
||||
);
|
||||
const otherResourceClientIds =
|
||||
clientsFromOtherResourcesBySite.get(siteId) ?? new Set<number>();
|
||||
|
||||
const clientSitesToInsert = clientSitesToAdd.map((clientId) => ({
|
||||
clientId,
|
||||
siteId
|
||||
}));
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} otherResourceClientIds=[${[...otherResourceClientIds].join(", ")}] mergedAllClientIds=[${mergedAllClientIds.join(", ")}]`
|
||||
);
|
||||
|
||||
// Expected clients from this resource are site-scoped: if this site is
|
||||
// no longer attached to the resource, the expected set is empty.
|
||||
const expectedClientIdsForSite = currentSiteIdSet.has(siteId)
|
||||
? mergedAllClientIds
|
||||
: [];
|
||||
|
||||
const clientSitesToAdd = expectedClientIdsForSite.filter(
|
||||
(clientId) =>
|
||||
!existingClientSiteIds.includes(clientId) &&
|
||||
!otherResourceClientIds.has(clientId) // dont add if already connected via another site resource
|
||||
);
|
||||
|
||||
const clientSitesToInsert = clientSitesToAdd.map((clientId) => ({
|
||||
clientId,
|
||||
siteId
|
||||
}));
|
||||
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} clientSites toAdd=[${clientSitesToAdd.join(", ")}]`
|
||||
);
|
||||
|
||||
if (clientSitesToInsert.length > 0) {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} clientSites toAdd=[${clientSitesToAdd.join(", ")}]`
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} inserting ${clientSitesToInsert.length} clientSite association(s)`
|
||||
);
|
||||
|
||||
if (clientSitesToInsert.length > 0) {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} inserting ${clientSitesToInsert.length} clientSite association(s)`
|
||||
);
|
||||
await trx
|
||||
.insert(clientSitesAssociationsCache)
|
||||
.values(clientSitesToInsert)
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} inserted clientSite associations`
|
||||
);
|
||||
} else {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} no clientSite associations to insert`
|
||||
);
|
||||
}
|
||||
|
||||
// Now remove any client-site associations that should no longer exist
|
||||
const clientSitesToRemove = existingClientSiteIds.filter(
|
||||
(clientId) =>
|
||||
!expectedClientIdsForSite.includes(clientId) &&
|
||||
!otherResourceClientIds.has(clientId) // dont remove if there is still another connection for another site resource
|
||||
);
|
||||
|
||||
await trx
|
||||
.insert(clientSitesAssociationsCache)
|
||||
.values(clientSitesToInsert)
|
||||
.returning();
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} clientSites toRemove=[${clientSitesToRemove.join(", ")}]`
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} inserted clientSite associations`
|
||||
);
|
||||
|
||||
if (clientSitesToRemove.length > 0) {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} deleting ${clientSitesToRemove.length} clientSite association(s)`
|
||||
);
|
||||
await trx
|
||||
.delete(clientSitesAssociationsCache)
|
||||
.where(
|
||||
and(
|
||||
eq(clientSitesAssociationsCache.siteId, siteId),
|
||||
inArray(
|
||||
clientSitesAssociationsCache.clientId,
|
||||
clientSitesToRemove
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Now handle the messages to add/remove peers on both the newt and olm sides
|
||||
await handleMessagesForSiteClients(
|
||||
site,
|
||||
siteId,
|
||||
mergedAllClients,
|
||||
existingClients,
|
||||
clientSitesToAdd,
|
||||
clientSitesToRemove,
|
||||
trx
|
||||
} else {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} no clientSite associations to insert`
|
||||
);
|
||||
} catch (err) {
|
||||
// Don't let a failure on one site abort processing of every
|
||||
// other site queued after it in this run. Since we're not
|
||||
// re-throwing, the outer wrapper's retry/requeue logic never
|
||||
// sees this failure, so explicitly queue this resource for a
|
||||
// follow-up pass to reconcile whatever this site didn't get to.
|
||||
logger.error(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} failed while processing site for siteResourceId=${siteResource.siteResourceId}, continuing with remaining sites and queuing a follow-up pass:`,
|
||||
err
|
||||
);
|
||||
await rebuildQueue.enqueue({
|
||||
type: "site-resource",
|
||||
id: siteResource.siteResourceId
|
||||
});
|
||||
}
|
||||
|
||||
// Now remove any client-site associations that should no longer exist
|
||||
const clientSitesToRemove = existingClientSiteIds.filter(
|
||||
(clientId) =>
|
||||
!expectedClientIdsForSite.includes(clientId) &&
|
||||
!otherResourceClientIds.has(clientId) // dont remove if there is still another connection for another site resource
|
||||
);
|
||||
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} clientSites toRemove=[${clientSitesToRemove.join(", ")}]`
|
||||
);
|
||||
|
||||
if (clientSitesToRemove.length > 0) {
|
||||
logger.debug(
|
||||
`rebuildClientAssociations: [rebuildClientAssociationsFromSiteResource] siteId=${siteId} deleting ${clientSitesToRemove.length} clientSite association(s)`
|
||||
);
|
||||
await trx
|
||||
.delete(clientSitesAssociationsCache)
|
||||
.where(
|
||||
and(
|
||||
eq(clientSitesAssociationsCache.siteId, siteId),
|
||||
inArray(
|
||||
clientSitesAssociationsCache.clientId,
|
||||
clientSitesToRemove
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Now handle the messages to add/remove peers on both the newt and olm sides
|
||||
await handleMessagesForSiteClients(
|
||||
site,
|
||||
siteId,
|
||||
mergedAllClients,
|
||||
existingClients,
|
||||
clientSitesToAdd,
|
||||
clientSitesToRemove,
|
||||
trx
|
||||
);
|
||||
}
|
||||
|
||||
// Handle subnet proxy target updates for the resource associations
|
||||
@@ -707,7 +657,7 @@ async function handleMessagesForSiteClients(
|
||||
trx: Transaction | typeof db = db
|
||||
): Promise<void> {
|
||||
if (!site.exitNodeId) {
|
||||
logger.debug(
|
||||
logger.warn(
|
||||
`Exit node ID not on site ${site.siteId} so there is no reason to update clients because it must be offline`
|
||||
);
|
||||
return;
|
||||
@@ -721,14 +671,14 @@ async function handleMessagesForSiteClients(
|
||||
.limit(1);
|
||||
|
||||
if (!exitNode) {
|
||||
logger.debug(
|
||||
logger.warn(
|
||||
`Exit node not found for site ${site.siteId} so there is no reason to update clients because it must be offline`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!site.publicKey) {
|
||||
logger.debug(
|
||||
logger.warn(
|
||||
`Site publicKey not set for site ${site.siteId} so cannot add peers to clients`
|
||||
);
|
||||
return;
|
||||
@@ -742,7 +692,7 @@ async function handleMessagesForSiteClients(
|
||||
.where(eq(newts.siteId, siteId))
|
||||
.limit(1);
|
||||
if (!newt) {
|
||||
logger.debug(
|
||||
logger.warn(
|
||||
`Newt not found for site ${siteId} so cannot add peers to clients`
|
||||
);
|
||||
return;
|
||||
@@ -967,7 +917,7 @@ export async function updateClientSiteDestinations(
|
||||
|
||||
for (const site of sitesData) {
|
||||
if (!site.sites.subnet) {
|
||||
logger.debug(`Site ${site.sites.siteId} has no subnet, skipping`);
|
||||
logger.warn(`Site ${site.sites.siteId} has no subnet, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1706,17 +1656,10 @@ export async function rebuildClientAssociationsFromClient(
|
||||
await incrementOrgRebuildCount(client.orgId);
|
||||
try {
|
||||
const trx = primaryDb;
|
||||
// The whole locked rebuild is idempotent (it diffs full expected vs.
|
||||
// actual state each time), so on a transient DB error it's safe to
|
||||
// retry the entire thing rather than just the failed query.
|
||||
return await withRetry(
|
||||
() =>
|
||||
lockManager.withLock(
|
||||
`rebuild-client-associations:client:${client.clientId}`,
|
||||
() => rebuildClientAssociationsFromClientImpl(client, trx),
|
||||
REBUILD_ASSOCIATIONS_LOCK_TTL_MS
|
||||
),
|
||||
`rebuildClientAssociationsFromClient:${client.clientId}`
|
||||
return await lockManager.withLock(
|
||||
`rebuild-client-associations:client:${client.clientId}`,
|
||||
() => rebuildClientAssociationsFromClientImpl(client, trx),
|
||||
REBUILD_ASSOCIATIONS_LOCK_TTL_MS
|
||||
);
|
||||
} catch (err: any) {
|
||||
if (
|
||||
@@ -1732,17 +1675,6 @@ export async function rebuildClientAssociationsFromClient(
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (isTransientError(err)) {
|
||||
logger.warn(
|
||||
`rebuildClientAssociations: transient DB error rebuilding client ${client.clientId} persisted after retries, queuing for deferred processing:`,
|
||||
err
|
||||
);
|
||||
await rebuildQueue.enqueue({
|
||||
type: "client",
|
||||
id: client.clientId
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
await decrementOrgRebuildCount(client.orgId);
|
||||
@@ -1894,15 +1826,12 @@ async function rebuildClientAssociationsFromClientImpl(
|
||||
|
||||
// Insert new associations
|
||||
if (resourcesToAdd.length > 0) {
|
||||
await trx
|
||||
.insert(clientSiteResourcesAssociationsCache)
|
||||
.values(
|
||||
resourcesToAdd.map((siteResourceId) => ({
|
||||
clientId: client.clientId,
|
||||
siteResourceId
|
||||
}))
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
await trx.insert(clientSiteResourcesAssociationsCache).values(
|
||||
resourcesToAdd.map((siteResourceId) => ({
|
||||
clientId: client.clientId,
|
||||
siteResourceId
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// Remove old associations
|
||||
@@ -1940,15 +1869,12 @@ async function rebuildClientAssociationsFromClientImpl(
|
||||
|
||||
// Insert new site associations
|
||||
if (sitesToAdd.length > 0) {
|
||||
await trx
|
||||
.insert(clientSitesAssociationsCache)
|
||||
.values(
|
||||
sitesToAdd.map((siteId) => ({
|
||||
clientId: client.clientId,
|
||||
siteId
|
||||
}))
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
await trx.insert(clientSitesAssociationsCache).values(
|
||||
sitesToAdd.map((siteId) => ({
|
||||
clientId: client.clientId,
|
||||
siteId
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// Remove old site associations
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
import logger from "@server/logger";
|
||||
import { isTransientError } from "@server/lib/dbRetry";
|
||||
|
||||
export type RebuildJobType = "site-resource" | "client";
|
||||
|
||||
export interface RebuildJob {
|
||||
type: RebuildJobType;
|
||||
id: number;
|
||||
// Number of times this job has already been re-queued after a transient
|
||||
// failure. Absent/0 means it has not failed yet.
|
||||
attempt?: number;
|
||||
}
|
||||
|
||||
export interface RebuildJobHandlers {
|
||||
@@ -28,10 +24,6 @@ export interface RebuildQueueManager {
|
||||
// retried shortly after against fresh DB state.
|
||||
const POLL_INTERVAL_MS = 500;
|
||||
const BATCH_SIZE = 5;
|
||||
// A job that fails with a transient DB error gets re-queued with backoff
|
||||
// instead of being dropped, up to this many times.
|
||||
const MAX_JOB_ATTEMPTS = 5;
|
||||
const JOB_RETRY_BASE_DELAY_MS = 1000;
|
||||
|
||||
function dedupeKey(job: RebuildJob): string {
|
||||
return `${job.type}:${job.id}`;
|
||||
@@ -114,29 +106,10 @@ class InMemoryRebuildQueue implements RebuildQueueManager {
|
||||
`Rebuild queue: completed ${job.type}:${job.id}`
|
||||
);
|
||||
} catch (err) {
|
||||
const attempt = (job.attempt ?? 0) + 1;
|
||||
if (isTransientError(err) && attempt <= MAX_JOB_ATTEMPTS) {
|
||||
const delay =
|
||||
JOB_RETRY_BASE_DELAY_MS * Math.pow(2, attempt - 1);
|
||||
logger.warn(
|
||||
`Rebuild queue: job ${job.type}:${job.id} hit a transient error (attempt ${attempt}/${MAX_JOB_ATTEMPTS}), re-queuing in ${delay}ms:`,
|
||||
err
|
||||
);
|
||||
setTimeout(() => {
|
||||
this.enqueue({ ...job, attempt }).catch(
|
||||
(enqueueErr) =>
|
||||
logger.error(
|
||||
`Rebuild queue: failed to re-queue ${job.type}:${job.id} after transient error:`,
|
||||
enqueueErr
|
||||
)
|
||||
);
|
||||
}, delay);
|
||||
} else {
|
||||
logger.error(
|
||||
`Rebuild queue: job ${job.type}:${job.id} threw an error:`,
|
||||
err
|
||||
);
|
||||
}
|
||||
logger.error(
|
||||
`Rebuild queue: job ${job.type}:${job.id} threw an error:`,
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
import { redis } from "#private/lib/redis";
|
||||
import logger from "@server/logger";
|
||||
|
||||
export const ORG_REBUILD_CONCURRENCY_LIMIT = 10;
|
||||
export const ORG_REBUILD_CONCURRENCY_LIMIT = 5;
|
||||
|
||||
// Safety-net TTL: slightly longer than the rebuild lock TTL (120 s). If a
|
||||
// server process dies while holding a rebuild, this ensures the counter key
|
||||
|
||||
@@ -14,16 +14,12 @@
|
||||
import { redis } from "#private/lib/redis";
|
||||
import { lockManager } from "#private/lib/lock";
|
||||
import logger from "@server/logger";
|
||||
import { isTransientError } from "@server/lib/dbRetry";
|
||||
|
||||
export type RebuildJobType = "site-resource" | "client";
|
||||
|
||||
export interface RebuildJob {
|
||||
type: RebuildJobType;
|
||||
id: number;
|
||||
// Number of times this job has already been re-queued after a transient
|
||||
// failure. Absent/0 means it has not failed yet.
|
||||
attempt?: number;
|
||||
}
|
||||
|
||||
export interface RebuildJobHandlers {
|
||||
@@ -47,11 +43,6 @@ const PROCESSOR_LOCK_TTL_MS = 120000 * BATCH_SIZE + 30000; // ~630 s
|
||||
|
||||
const POLL_INTERVAL_MS = 500;
|
||||
|
||||
// A job that fails with a transient DB error gets re-queued with backoff
|
||||
// instead of being dropped, up to this many times.
|
||||
const MAX_JOB_ATTEMPTS = 5;
|
||||
const JOB_RETRY_BASE_DELAY_MS = 1000;
|
||||
|
||||
class RedisRebuildQueue {
|
||||
private processingStarted = false;
|
||||
|
||||
@@ -189,33 +180,10 @@ class RedisRebuildQueue {
|
||||
`Rebuild queue: completed ${job.type}:${job.id}`
|
||||
);
|
||||
} catch (err) {
|
||||
const attempt = (job.attempt ?? 0) + 1;
|
||||
if (
|
||||
isTransientError(err) &&
|
||||
attempt <= MAX_JOB_ATTEMPTS
|
||||
) {
|
||||
const delay =
|
||||
JOB_RETRY_BASE_DELAY_MS *
|
||||
Math.pow(2, attempt - 1);
|
||||
logger.warn(
|
||||
`Rebuild queue: job ${job.type}:${job.id} hit a transient error (attempt ${attempt}/${MAX_JOB_ATTEMPTS}), re-queuing in ${delay}ms:`,
|
||||
err
|
||||
);
|
||||
setTimeout(() => {
|
||||
this.enqueue({ ...job, attempt }).catch(
|
||||
(enqueueErr) =>
|
||||
logger.error(
|
||||
`Rebuild queue: failed to re-queue ${job.type}:${job.id} after transient error:`,
|
||||
enqueueErr
|
||||
)
|
||||
);
|
||||
}, delay);
|
||||
} else {
|
||||
logger.error(
|
||||
`Rebuild queue: job ${job.type}:${job.id} threw an error:`,
|
||||
err
|
||||
);
|
||||
}
|
||||
logger.error(
|
||||
`Rebuild queue: job ${job.type}:${job.id} threw an error:`,
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -894,19 +894,6 @@ class RegionalRedisManager {
|
||||
return opts;
|
||||
}
|
||||
|
||||
// The regional Redis StatefulSet's "redis" service pins to pod redis-0
|
||||
// (primary). The replica (redis-1) is only reachable through the
|
||||
// per-pod headless service: <svc>.<namespace>.svc.cluster.local ->
|
||||
// redis-1.redis-headless.<namespace>.svc.cluster.local. Returns null
|
||||
// if the configured host doesn't match that pattern (e.g. local dev),
|
||||
// in which case callers should fall back to the primary for reads.
|
||||
private getReplicaHost(primaryHost: string): string | null {
|
||||
const match = primaryHost.match(/^redis\.([^.]+)\.svc\.cluster\.local$/);
|
||||
if (!match) return null;
|
||||
const namespace = match[1];
|
||||
return `redis-1.redis-headless.${namespace}.svc.cluster.local`;
|
||||
}
|
||||
|
||||
private initializeClients(): void {
|
||||
const cfg = this.getConfig();
|
||||
const baseOpts = {
|
||||
@@ -920,42 +907,35 @@ class RegionalRedisManager {
|
||||
|
||||
try {
|
||||
this.writeClient = new Redis(baseOpts);
|
||||
// redis-1 (replica) handles reads; fall back to primary if not resolvable
|
||||
this.readClient = new Redis({
|
||||
...baseOpts,
|
||||
host: cfg.host!.replace(/^(.*?)(\.\S+)$/, (_, h, rest) => {
|
||||
// Derive replica hostname from the headless service pattern:
|
||||
// redis.redis.svc.cluster.local -> redis-1.redis-headless.redis.svc.cluster.local
|
||||
// If it doesn't look like a k8s service, just use the same host
|
||||
return h + rest;
|
||||
})
|
||||
});
|
||||
|
||||
const replicaHost = this.getReplicaHost(cfg.host!);
|
||||
this.readClient = replicaHost
|
||||
? new Redis({ ...baseOpts, host: replicaHost })
|
||||
: this.writeClient;
|
||||
// For simplicity use same host for both; callers can always read from primary
|
||||
// The real replica routing is handled by the StatefulSet headless service
|
||||
this.readClient = this.writeClient;
|
||||
|
||||
this.writeClient.on("ready", () => {
|
||||
logger.info("Regional Redis write client ready");
|
||||
logger.info("Regional Redis client ready");
|
||||
this.isHealthy = true;
|
||||
});
|
||||
this.writeClient.on("error", (err) => {
|
||||
logger.error("Regional Redis write client error:", err);
|
||||
logger.error("Regional Redis client error:", err);
|
||||
this.isHealthy = false;
|
||||
});
|
||||
this.writeClient.on("reconnecting", () => {
|
||||
logger.info("Regional Redis write client reconnecting...");
|
||||
logger.info("Regional Redis client reconnecting...");
|
||||
this.isHealthy = false;
|
||||
});
|
||||
|
||||
if (this.readClient !== this.writeClient) {
|
||||
this.readClient.on("ready", () => {
|
||||
logger.info("Regional Redis read client ready");
|
||||
});
|
||||
this.readClient.on("error", (err) => {
|
||||
logger.error("Regional Redis read client error:", err);
|
||||
});
|
||||
this.readClient.on("reconnecting", () => {
|
||||
logger.info("Regional Redis read client reconnecting...");
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
replicaHost
|
||||
? `Regional Redis client initialized (reads routed to replica ${replicaHost})`
|
||||
: "Regional Redis client initialized (no replica resolvable, reads routed to primary)"
|
||||
);
|
||||
logger.info("Regional Redis client initialized");
|
||||
} catch (error) {
|
||||
logger.error("Failed to initialize regional Redis client:", error);
|
||||
this.isEnabled = false;
|
||||
@@ -1061,14 +1041,11 @@ class RegionalRedisManager {
|
||||
|
||||
public async disconnect(): Promise<void> {
|
||||
try {
|
||||
if (this.readClient && this.readClient !== this.writeClient) {
|
||||
await this.readClient.quit();
|
||||
}
|
||||
this.readClient = null;
|
||||
if (this.writeClient) {
|
||||
await this.writeClient.quit();
|
||||
this.writeClient = null;
|
||||
}
|
||||
this.readClient = null;
|
||||
logger.info("Regional Redis client disconnected");
|
||||
} catch (error) {
|
||||
logger.error("Error disconnecting regional Redis client:", error);
|
||||
|
||||
@@ -104,18 +104,6 @@ export async function getOrgUsage(
|
||||
orgId,
|
||||
LimitId.ORGANIZATIONS
|
||||
);
|
||||
const publicResources = await usageService.getUsage(
|
||||
orgId,
|
||||
LimitId.PUBLIC_RESOURCES
|
||||
);
|
||||
const privateResources = await usageService.getUsage(
|
||||
orgId,
|
||||
LimitId.PRIVATE_RESOURCES
|
||||
);
|
||||
const machineClients = await usageService.getUsage(
|
||||
orgId,
|
||||
LimitId.MACHINE_CLIENTS
|
||||
);
|
||||
// const egressData = await usageService.getUsage(
|
||||
// orgId,
|
||||
// FeatureId.EGRESS_DATA_MB
|
||||
@@ -139,15 +127,6 @@ export async function getOrgUsage(
|
||||
if (organizations) {
|
||||
usageData.push(organizations);
|
||||
}
|
||||
if (publicResources) {
|
||||
usageData.push(publicResources);
|
||||
}
|
||||
if (privateResources) {
|
||||
usageData.push(privateResources);
|
||||
}
|
||||
if (machineClients) {
|
||||
usageData.push(machineClients);
|
||||
}
|
||||
|
||||
const orgLimits = await db
|
||||
.select()
|
||||
|
||||
@@ -29,41 +29,37 @@ export async function createExitNode(
|
||||
.where(eq(exitNodes.publicKey, publicKey));
|
||||
let exitNode: ExitNode;
|
||||
if (!exitNodeQuery) {
|
||||
const { value: address, release } = await getNextAvailableSubnet();
|
||||
try {
|
||||
// TODO: eventually we will want to get the next available port so that we can multiple exit nodes
|
||||
// const listenPort = await getNextAvailablePort();
|
||||
const listenPort = config.getRawConfig().gerbil.start_port;
|
||||
let subEndpoint = "";
|
||||
if (config.getRawConfig().gerbil.use_subdomain) {
|
||||
subEndpoint = await getUniqueExitNodeEndpointName();
|
||||
}
|
||||
|
||||
const exitNodeName =
|
||||
config.getRawConfig().gerbil.exit_node_name ||
|
||||
`Exit Node ${publicKey.slice(0, 8)}`;
|
||||
|
||||
// create a new exit node
|
||||
[exitNode] = await db
|
||||
.insert(exitNodes)
|
||||
.values({
|
||||
publicKey,
|
||||
endpoint: `${subEndpoint}${subEndpoint != "" ? "." : ""}${config.getRawConfig().gerbil.base_endpoint}`,
|
||||
address,
|
||||
listenPort,
|
||||
online: true,
|
||||
reachableAt,
|
||||
name: exitNodeName
|
||||
})
|
||||
.returning()
|
||||
.execute();
|
||||
|
||||
logger.info(
|
||||
`Created new exit node ${exitNode.name} with address ${exitNode.address} and port ${exitNode.listenPort}`
|
||||
);
|
||||
} finally {
|
||||
await release();
|
||||
const address = await getNextAvailableSubnet();
|
||||
// TODO: eventually we will want to get the next available port so that we can multiple exit nodes
|
||||
// const listenPort = await getNextAvailablePort();
|
||||
const listenPort = config.getRawConfig().gerbil.start_port;
|
||||
let subEndpoint = "";
|
||||
if (config.getRawConfig().gerbil.use_subdomain) {
|
||||
subEndpoint = await getUniqueExitNodeEndpointName();
|
||||
}
|
||||
|
||||
const exitNodeName =
|
||||
config.getRawConfig().gerbil.exit_node_name ||
|
||||
`Exit Node ${publicKey.slice(0, 8)}`;
|
||||
|
||||
// create a new exit node
|
||||
[exitNode] = await db
|
||||
.insert(exitNodes)
|
||||
.values({
|
||||
publicKey,
|
||||
endpoint: `${subEndpoint}${subEndpoint != "" ? "." : ""}${config.getRawConfig().gerbil.base_endpoint}`,
|
||||
address,
|
||||
listenPort,
|
||||
online: true,
|
||||
reachableAt,
|
||||
name: exitNodeName
|
||||
})
|
||||
.returning()
|
||||
.execute();
|
||||
|
||||
logger.info(
|
||||
`Created new exit node ${exitNode.name} with address ${exitNode.address} and port ${exitNode.listenPort}`
|
||||
);
|
||||
} else {
|
||||
// update the reachable at
|
||||
[exitNode] = await db
|
||||
|
||||
@@ -114,6 +114,8 @@ export async function createRemoteExitNode(
|
||||
}
|
||||
|
||||
const secretHash = await hashPassword(secret);
|
||||
// const address = await getNextAvailableSubnet();
|
||||
const address = "100.89.140.1/24"; // FOR NOW LETS HARDCODE THESE ADDRESSES
|
||||
|
||||
const [existingRemoteExitNode] = await db
|
||||
.select()
|
||||
@@ -189,106 +191,89 @@ export async function createRemoteExitNode(
|
||||
);
|
||||
}
|
||||
|
||||
// If this remote exit node isn't already backing an exit node in
|
||||
// another org, we're about to create a brand new one. Reserve a
|
||||
// subnet for it up front so the allocation lock is held across the
|
||||
// whole insert - this guarantees exit node subnets never overlap,
|
||||
// even under concurrent creation, which matters for HA setups.
|
||||
let releaseSubnetLock: (() => Promise<void>) | null = null;
|
||||
let newExitNodeAddress: string | null = null;
|
||||
if (!existingExitNode) {
|
||||
const { value, release } = await getNextAvailableSubnet();
|
||||
newExitNodeAddress = value;
|
||||
releaseSubnetLock = release;
|
||||
}
|
||||
await db.transaction(async (trx) => {
|
||||
if (!existingExitNode) {
|
||||
const [res] = await trx
|
||||
.insert(exitNodes)
|
||||
.values({
|
||||
name: remoteExitNodeId,
|
||||
address,
|
||||
endpoint: "",
|
||||
publicKey: "",
|
||||
listenPort: 0,
|
||||
online: false,
|
||||
type: "remoteExitNode"
|
||||
})
|
||||
.returning();
|
||||
existingExitNode = res;
|
||||
}
|
||||
|
||||
try {
|
||||
await db.transaction(async (trx) => {
|
||||
if (!existingExitNode) {
|
||||
const [res] = await trx
|
||||
.insert(exitNodes)
|
||||
.values({
|
||||
name: remoteExitNodeId,
|
||||
address: newExitNodeAddress!,
|
||||
endpoint: "",
|
||||
publicKey: "",
|
||||
listenPort: 0,
|
||||
online: false,
|
||||
type: "remoteExitNode"
|
||||
})
|
||||
.returning();
|
||||
existingExitNode = res;
|
||||
}
|
||||
|
||||
if (!existingRemoteExitNode) {
|
||||
await trx.insert(remoteExitNodes).values({
|
||||
remoteExitNodeId: remoteExitNodeId,
|
||||
secretHash,
|
||||
dateCreated: moment().toISOString(),
|
||||
if (!existingRemoteExitNode) {
|
||||
await trx.insert(remoteExitNodes).values({
|
||||
remoteExitNodeId: remoteExitNodeId,
|
||||
secretHash,
|
||||
dateCreated: moment().toISOString(),
|
||||
exitNodeId: existingExitNode.exitNodeId
|
||||
});
|
||||
} else {
|
||||
// update the existing remote exit node
|
||||
await trx
|
||||
.update(remoteExitNodes)
|
||||
.set({
|
||||
exitNodeId: existingExitNode.exitNodeId
|
||||
});
|
||||
} else {
|
||||
// update the existing remote exit node
|
||||
await trx
|
||||
.update(remoteExitNodes)
|
||||
.set({
|
||||
exitNodeId: existingExitNode.exitNodeId
|
||||
})
|
||||
.where(
|
||||
})
|
||||
.where(
|
||||
eq(
|
||||
remoteExitNodes.remoteExitNodeId,
|
||||
existingRemoteExitNode.remoteExitNodeId
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (!existingExitNodeOrg) {
|
||||
await trx.insert(exitNodeOrgs).values({
|
||||
exitNodeId: existingExitNode.exitNodeId,
|
||||
orgId: orgId
|
||||
});
|
||||
}
|
||||
|
||||
// calculate if the node is in any other of the orgs before we count it as an add to the billing org
|
||||
if (org.billingOrgId) {
|
||||
const otherBillingOrgs = await trx
|
||||
.select()
|
||||
.from(orgs)
|
||||
.where(
|
||||
and(
|
||||
eq(orgs.billingOrgId, org.billingOrgId),
|
||||
ne(orgs.orgId, orgId)
|
||||
)
|
||||
);
|
||||
|
||||
const billingOrgIds = otherBillingOrgs.map((o) => o.orgId);
|
||||
|
||||
const orgsInBillingDomainThatTheNodeIsStillIn = await trx
|
||||
.select()
|
||||
.from(exitNodeOrgs)
|
||||
.where(
|
||||
and(
|
||||
eq(
|
||||
remoteExitNodes.remoteExitNodeId,
|
||||
existingRemoteExitNode.remoteExitNodeId
|
||||
)
|
||||
);
|
||||
exitNodeOrgs.exitNodeId,
|
||||
existingExitNode.exitNodeId
|
||||
),
|
||||
inArray(exitNodeOrgs.orgId, billingOrgIds)
|
||||
)
|
||||
);
|
||||
|
||||
if (orgsInBillingDomainThatTheNodeIsStillIn.length === 0) {
|
||||
await usageService.add(
|
||||
orgId,
|
||||
LimitId.REMOTE_EXIT_NODES,
|
||||
1,
|
||||
trx
|
||||
);
|
||||
}
|
||||
|
||||
if (!existingExitNodeOrg) {
|
||||
await trx.insert(exitNodeOrgs).values({
|
||||
exitNodeId: existingExitNode.exitNodeId,
|
||||
orgId: orgId
|
||||
});
|
||||
}
|
||||
|
||||
// calculate if the node is in any other of the orgs before we count it as an add to the billing org
|
||||
if (org.billingOrgId) {
|
||||
const otherBillingOrgs = await trx
|
||||
.select()
|
||||
.from(orgs)
|
||||
.where(
|
||||
and(
|
||||
eq(orgs.billingOrgId, org.billingOrgId),
|
||||
ne(orgs.orgId, orgId)
|
||||
)
|
||||
);
|
||||
|
||||
const billingOrgIds = otherBillingOrgs.map((o) => o.orgId);
|
||||
|
||||
const orgsInBillingDomainThatTheNodeIsStillIn = await trx
|
||||
.select()
|
||||
.from(exitNodeOrgs)
|
||||
.where(
|
||||
and(
|
||||
eq(
|
||||
exitNodeOrgs.exitNodeId,
|
||||
existingExitNode.exitNodeId
|
||||
),
|
||||
inArray(exitNodeOrgs.orgId, billingOrgIds)
|
||||
)
|
||||
);
|
||||
|
||||
if (orgsInBillingDomainThatTheNodeIsStillIn.length === 0) {
|
||||
await usageService.add(
|
||||
orgId,
|
||||
LimitId.REMOTE_EXIT_NODES,
|
||||
1,
|
||||
trx
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await releaseSubnetLock?.();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const token = generateSessionToken();
|
||||
await createRemoteExitNodeSession(token, remoteExitNodeId);
|
||||
|
||||
@@ -13,41 +13,37 @@ export async function createExitNode(
|
||||
const [exitNodeQuery] = await db.select().from(exitNodes).limit(1);
|
||||
let exitNode: ExitNode;
|
||||
if (!exitNodeQuery) {
|
||||
const { value: address, release } = await getNextAvailableSubnet();
|
||||
try {
|
||||
// TODO: eventually we will want to get the next available port so that we can multiple exit nodes
|
||||
// const listenPort = await getNextAvailablePort();
|
||||
const listenPort = config.getRawConfig().gerbil.start_port;
|
||||
let subEndpoint = "";
|
||||
if (config.getRawConfig().gerbil.use_subdomain) {
|
||||
subEndpoint = await getUniqueExitNodeEndpointName();
|
||||
}
|
||||
|
||||
const exitNodeName =
|
||||
config.getRawConfig().gerbil.exit_node_name ||
|
||||
`Exit Node ${publicKey.slice(0, 8)}`;
|
||||
|
||||
// create a new exit node
|
||||
[exitNode] = await db
|
||||
.insert(exitNodes)
|
||||
.values({
|
||||
publicKey,
|
||||
endpoint: `${subEndpoint}${subEndpoint != "" ? "." : ""}${config.getRawConfig().gerbil.base_endpoint}`,
|
||||
address,
|
||||
online: true,
|
||||
listenPort,
|
||||
reachableAt,
|
||||
name: exitNodeName
|
||||
})
|
||||
.returning()
|
||||
.execute();
|
||||
|
||||
logger.info(
|
||||
`Created new exit node ${exitNode.name} with address ${exitNode.address} and port ${exitNode.listenPort}`
|
||||
);
|
||||
} finally {
|
||||
await release();
|
||||
const address = await getNextAvailableSubnet();
|
||||
// TODO: eventually we will want to get the next available port so that we can multiple exit nodes
|
||||
// const listenPort = await getNextAvailablePort();
|
||||
const listenPort = config.getRawConfig().gerbil.start_port;
|
||||
let subEndpoint = "";
|
||||
if (config.getRawConfig().gerbil.use_subdomain) {
|
||||
subEndpoint = await getUniqueExitNodeEndpointName();
|
||||
}
|
||||
|
||||
const exitNodeName =
|
||||
config.getRawConfig().gerbil.exit_node_name ||
|
||||
`Exit Node ${publicKey.slice(0, 8)}`;
|
||||
|
||||
// create a new exit node
|
||||
[exitNode] = await db
|
||||
.insert(exitNodes)
|
||||
.values({
|
||||
publicKey,
|
||||
endpoint: `${subEndpoint}${subEndpoint != "" ? "." : ""}${config.getRawConfig().gerbil.base_endpoint}`,
|
||||
address,
|
||||
online: true,
|
||||
listenPort,
|
||||
reachableAt,
|
||||
name: exitNodeName
|
||||
})
|
||||
.returning()
|
||||
.execute();
|
||||
|
||||
logger.info(
|
||||
`Created new exit node ${exitNode.name} with address ${exitNode.address} and port ${exitNode.listenPort}`
|
||||
);
|
||||
} else {
|
||||
// update the existing exit node
|
||||
[exitNode] = await db
|
||||
|
||||
@@ -52,13 +52,13 @@ export async function buildClientConfigurationForNewtClient(
|
||||
clientsRes
|
||||
.filter((client) => {
|
||||
if (!client.clients.pubKey) {
|
||||
logger.debug(
|
||||
logger.warn(
|
||||
`Client ${client.clients.clientId} has no public key, skipping`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (!client.clients.subnet) {
|
||||
logger.debug(
|
||||
logger.warn(
|
||||
`Client ${client.clients.clientId} has no subnet, skipping`
|
||||
);
|
||||
return false;
|
||||
|
||||
@@ -19,7 +19,7 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (
|
||||
}
|
||||
|
||||
if (!newt.siteId) {
|
||||
logger.warn("Newt has no site ID!");
|
||||
logger.warn("Newt has no client ID!");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -34,12 +34,6 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (
|
||||
.where(eq(sites.siteId, newt.siteId!))
|
||||
.returning();
|
||||
|
||||
if (!site) {
|
||||
throw new Error(
|
||||
`Could not find site ${newt.siteId} to update disconnection from disconnect message`
|
||||
);
|
||||
}
|
||||
|
||||
await fireSiteOfflineAlert(
|
||||
site.orgId,
|
||||
site.siteId,
|
||||
@@ -49,6 +43,6 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (
|
||||
);
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("Error handling site disconnecting message", error);
|
||||
logger.error("Error handling disconnecting message", { error });
|
||||
}
|
||||
};
|
||||
|
||||
@@ -5,21 +5,8 @@ import { Newt } from "@server/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import logger from "@server/logger";
|
||||
import { sendNewtSyncMessage } from "./sync";
|
||||
import semver from "semver";
|
||||
import { recordSitePing } from "./pingAccumulator";
|
||||
|
||||
const NEWT_SUPPORTS_SYNC_VERSION = ">=1.14.0";
|
||||
const PONG = {
|
||||
message: {
|
||||
type: "pong",
|
||||
data: {
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
},
|
||||
broadcast: false,
|
||||
excludeSender: false
|
||||
};
|
||||
|
||||
/**
|
||||
* Handles ping messages from newt clients.
|
||||
*
|
||||
@@ -50,14 +37,6 @@ export const handleNewtPingMessage: MessageHandler = async (context) => {
|
||||
// cross-region latency to the database.
|
||||
recordSitePing(newt.siteId);
|
||||
|
||||
if (
|
||||
newt.version &&
|
||||
!semver.satisfies(newt.version, NEWT_SUPPORTS_SYNC_VERSION)
|
||||
) {
|
||||
// Newt does not support the sync message so not checking - stop here -
|
||||
return PONG;
|
||||
}
|
||||
|
||||
// Check config version and sync if stale.
|
||||
const configVersion = await getClientConfigVersion(newt.newtId);
|
||||
|
||||
@@ -86,5 +65,14 @@ export const handleNewtPingMessage: MessageHandler = async (context) => {
|
||||
await sendNewtSyncMessage(newt, site);
|
||||
}
|
||||
|
||||
return PONG;
|
||||
return {
|
||||
message: {
|
||||
type: "pong",
|
||||
data: {
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
},
|
||||
broadcast: false,
|
||||
excludeSender: false
|
||||
};
|
||||
};
|
||||
|
||||
@@ -3,7 +3,6 @@ import { sites, clients, olms } from "@server/db";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import logger from "@server/logger";
|
||||
import { fireSiteOnlineAlert } from "@server/lib/alerts";
|
||||
import { withRetry } from "@server/lib/dbRetry";
|
||||
|
||||
/**
|
||||
* Ping Accumulator
|
||||
@@ -23,6 +22,8 @@ import { withRetry } from "@server/lib/dbRetry";
|
||||
*/
|
||||
|
||||
const FLUSH_INTERVAL_MS = 10_000; // Flush every 10 seconds
|
||||
const MAX_RETRIES = 5;
|
||||
const BASE_DELAY_MS = 50;
|
||||
|
||||
// ── Site (newt) pings ──────────────────────────────────────────────────
|
||||
// Map of siteId -> latest ping timestamp (unix seconds)
|
||||
@@ -265,7 +266,85 @@ export async function flushPingsToDb(): Promise<void> {
|
||||
}
|
||||
|
||||
// ── Retry / Error Helpers ──────────────────────────────────────────────
|
||||
// See @server/lib/dbRetry for the shared withRetry/isTransientError helpers.
|
||||
|
||||
/**
|
||||
* Simple retry wrapper with exponential backoff for transient errors
|
||||
* (deadlocks, connection timeouts, unexpected disconnects).
|
||||
*
|
||||
* PostgreSQL deadlocks (40P01) are always safe to retry: the database
|
||||
* guarantees exactly one winner per deadlock pair, so the loser just needs
|
||||
* to try again. MAX_RETRIES is intentionally higher than typical connection
|
||||
* retry budgets to give deadlock victims enough chances to succeed.
|
||||
*/
|
||||
async function withRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
context: string
|
||||
): Promise<T> {
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error: any) {
|
||||
if (isTransientError(error) && attempt < MAX_RETRIES) {
|
||||
attempt++;
|
||||
const baseDelay = Math.pow(2, attempt - 1) * BASE_DELAY_MS;
|
||||
const jitter = Math.random() * baseDelay;
|
||||
const delay = baseDelay + jitter;
|
||||
logger.warn(
|
||||
`Transient DB error in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`,
|
||||
{ code: error?.code ?? error?.cause?.code }
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect transient errors that are safe to retry.
|
||||
*/
|
||||
function isTransientError(error: any): boolean {
|
||||
if (!error) return false;
|
||||
|
||||
const message = (error.message || "").toLowerCase();
|
||||
const causeMessage = (error.cause?.message || "").toLowerCase();
|
||||
const code = error.code || error.cause?.code || "";
|
||||
|
||||
// Connection timeout / terminated
|
||||
if (
|
||||
message.includes("connection timeout") ||
|
||||
message.includes("connection terminated") ||
|
||||
message.includes("timeout exceeded when trying to connect") ||
|
||||
causeMessage.includes("connection terminated unexpectedly") ||
|
||||
causeMessage.includes("connection timeout")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// PostgreSQL deadlock detected - always safe to retry (one winner guaranteed)
|
||||
if (code === "40P01" || message.includes("deadlock")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// PostgreSQL serialization failure
|
||||
if (code === "40001") {
|
||||
return true;
|
||||
}
|
||||
|
||||
// ECONNRESET, ECONNREFUSED, EPIPE, ETIMEDOUT
|
||||
if (
|
||||
code === "ECONNRESET" ||
|
||||
code === "ECONNREFUSED" ||
|
||||
code === "EPIPE" ||
|
||||
code === "ETIMEDOUT"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── Lifecycle ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -9,45 +9,45 @@ import {
|
||||
import { canCompress } from "@server/lib/clientVersionChecks";
|
||||
|
||||
export async function sendNewtSyncMessage(newt: Newt, site: Site) {
|
||||
const {
|
||||
tcpTargets,
|
||||
udpTargets,
|
||||
validHealthCheckTargets,
|
||||
browserGatewayTargets,
|
||||
remoteExitNodeSubnets
|
||||
} = await buildTargetConfigurationForNewtClient(site.siteId);
|
||||
let exitNode: ExitNode | undefined;
|
||||
if (site.exitNodeId) {
|
||||
[exitNode] = await db
|
||||
.select()
|
||||
.from(exitNodes)
|
||||
.where(eq(exitNodes.exitNodeId, site.exitNodeId))
|
||||
.limit(1);
|
||||
}
|
||||
const { peers, targets } = await buildClientConfigurationForNewtClient(
|
||||
site,
|
||||
exitNode
|
||||
);
|
||||
await sendToClient(
|
||||
newt.newtId,
|
||||
{
|
||||
type: "newt/sync",
|
||||
data: {
|
||||
proxyTargets: {
|
||||
udp: udpTargets,
|
||||
tcp: tcpTargets
|
||||
},
|
||||
healthCheckTargets: validHealthCheckTargets,
|
||||
peers: peers,
|
||||
clientTargets: targets,
|
||||
browserGatewayTargets: browserGatewayTargets,
|
||||
remoteExitNodeSubnets: remoteExitNodeSubnets
|
||||
}
|
||||
},
|
||||
{
|
||||
compress: canCompress(newt.version, "newt")
|
||||
}
|
||||
).catch((error) => {
|
||||
logger.warn(`Error sending newt sync message:`, error);
|
||||
});
|
||||
// const {
|
||||
// tcpTargets,
|
||||
// udpTargets,
|
||||
// validHealthCheckTargets,
|
||||
// browserGatewayTargets,
|
||||
// remoteExitNodeSubnets
|
||||
// } = await buildTargetConfigurationForNewtClient(site.siteId);
|
||||
// let exitNode: ExitNode | undefined;
|
||||
// if (site.exitNodeId) {
|
||||
// [exitNode] = await db
|
||||
// .select()
|
||||
// .from(exitNodes)
|
||||
// .where(eq(exitNodes.exitNodeId, site.exitNodeId))
|
||||
// .limit(1);
|
||||
// }
|
||||
// const { peers, targets } = await buildClientConfigurationForNewtClient(
|
||||
// site,
|
||||
// exitNode
|
||||
// );
|
||||
// await sendToClient(
|
||||
// newt.newtId,
|
||||
// {
|
||||
// type: "newt/sync",
|
||||
// data: {
|
||||
// proxyTargets: {
|
||||
// udp: udpTargets,
|
||||
// tcp: tcpTargets
|
||||
// },
|
||||
// healthCheckTargets: validHealthCheckTargets,
|
||||
// peers: peers,
|
||||
// clientTargets: targets,
|
||||
// browserGatewayTargets: browserGatewayTargets,
|
||||
// remoteExitNodeSubnets: remoteExitNodeSubnets
|
||||
// }
|
||||
// },
|
||||
// {
|
||||
// compress: canCompress(newt.version, "newt")
|
||||
// }
|
||||
// ).catch((error) => {
|
||||
// logger.warn(`Error sending newt sync message:`, error);
|
||||
// });
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ export async function buildSiteConfigurationForOlmClient(
|
||||
}
|
||||
|
||||
if (!site.subnet) {
|
||||
logger.debug(`Site ${site.siteId} has no subnet, skipping`);
|
||||
logger.warn(`Site ${site.siteId} has no subnet, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,9 +6,7 @@ import logger from "@server/logger";
|
||||
/**
|
||||
* Handles disconnecting messages from clients to show disconnected in the ui
|
||||
*/
|
||||
export const handleOlmDisconnectingMessage: MessageHandler = async (
|
||||
context
|
||||
) => {
|
||||
export const handleOlmDisconnectingMessage: MessageHandler = async (context) => {
|
||||
const { message, client: c, sendToClient } = context;
|
||||
const olm = c as Olm;
|
||||
|
||||
@@ -31,6 +29,6 @@ export const handleOlmDisconnectingMessage: MessageHandler = async (
|
||||
})
|
||||
.where(eq(clients.clientId, olm.clientId));
|
||||
} catch (error) {
|
||||
logger.error("Error handling client disconnecting message", error);
|
||||
logger.error("Error handling disconnecting message", { error });
|
||||
}
|
||||
};
|
||||
|
||||
@@ -268,11 +268,7 @@ export async function createSite(
|
||||
|
||||
let newSite: Site | undefined;
|
||||
try {
|
||||
if (type === "wireguard" && subnet && exitNodeId) {
|
||||
// Only wireguard sites actually persist the provided subnet/exitNodeId.
|
||||
// Newt sites have their subnet/exit node chosen (under a lock) when the
|
||||
// newt connects, so validating them here is both unnecessary and racy,
|
||||
// since pickSiteDefaults does not lock the subnet it suggests.
|
||||
if (subnet && exitNodeId) {
|
||||
//make sure the subnet is in the range of the exit node if provided
|
||||
const [exitNode] = await db
|
||||
.select()
|
||||
|
||||
@@ -3,12 +3,8 @@ import { db, orgs } from "@server/db";
|
||||
import { actions, roles, roleActions } from "@server/db";
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import logger from "@server/logger";
|
||||
import { build } from "@server/build";
|
||||
|
||||
export async function ensureActions() {
|
||||
if (build == "saas") {
|
||||
return;
|
||||
}
|
||||
await db.transaction(async (trx) => {
|
||||
const actionIds = Object.values(ActionsEnum);
|
||||
const existingActions = await trx.select().from(actions).execute();
|
||||
|
||||
@@ -158,9 +158,6 @@ const tierLimits: Record<
|
||||
domains: number;
|
||||
remoteNodes: number;
|
||||
organizations: number;
|
||||
publicResources: number;
|
||||
privateResources: number;
|
||||
machineClients: number;
|
||||
}
|
||||
> = {
|
||||
basic: {
|
||||
@@ -168,50 +165,35 @@ const tierLimits: Record<
|
||||
sites: freeLimitSet[LimitId.SITES]?.value ?? 0,
|
||||
domains: freeLimitSet[LimitId.DOMAINS]?.value ?? 0,
|
||||
remoteNodes: freeLimitSet[LimitId.REMOTE_EXIT_NODES]?.value ?? 0,
|
||||
organizations: freeLimitSet[LimitId.ORGANIZATIONS]?.value ?? 0,
|
||||
publicResources: freeLimitSet[LimitId.PUBLIC_RESOURCES]?.value ?? 0,
|
||||
privateResources: freeLimitSet[LimitId.PRIVATE_RESOURCES]?.value ?? 0,
|
||||
machineClients: freeLimitSet[LimitId.MACHINE_CLIENTS]?.value ?? 0
|
||||
organizations: freeLimitSet[LimitId.ORGANIZATIONS]?.value ?? 0
|
||||
},
|
||||
tier1: {
|
||||
users: tier1LimitSet[LimitId.USERS]?.value ?? 0,
|
||||
sites: tier1LimitSet[LimitId.SITES]?.value ?? 0,
|
||||
domains: tier1LimitSet[LimitId.DOMAINS]?.value ?? 0,
|
||||
remoteNodes: tier1LimitSet[LimitId.REMOTE_EXIT_NODES]?.value ?? 0,
|
||||
organizations: tier1LimitSet[LimitId.ORGANIZATIONS]?.value ?? 0,
|
||||
publicResources: tier1LimitSet[LimitId.PUBLIC_RESOURCES]?.value ?? 0,
|
||||
privateResources: tier1LimitSet[LimitId.PRIVATE_RESOURCES]?.value ?? 0,
|
||||
machineClients: tier1LimitSet[LimitId.MACHINE_CLIENTS]?.value ?? 0
|
||||
organizations: tier1LimitSet[LimitId.ORGANIZATIONS]?.value ?? 0
|
||||
},
|
||||
tier2: {
|
||||
users: tier2LimitSet[LimitId.USERS]?.value ?? 0,
|
||||
sites: tier2LimitSet[LimitId.SITES]?.value ?? 0,
|
||||
domains: tier2LimitSet[LimitId.DOMAINS]?.value ?? 0,
|
||||
remoteNodes: tier2LimitSet[LimitId.REMOTE_EXIT_NODES]?.value ?? 0,
|
||||
organizations: tier2LimitSet[LimitId.ORGANIZATIONS]?.value ?? 0,
|
||||
publicResources: tier2LimitSet[LimitId.PUBLIC_RESOURCES]?.value ?? 0,
|
||||
privateResources: tier2LimitSet[LimitId.PRIVATE_RESOURCES]?.value ?? 0,
|
||||
machineClients: tier2LimitSet[LimitId.MACHINE_CLIENTS]?.value ?? 0
|
||||
organizations: tier2LimitSet[LimitId.ORGANIZATIONS]?.value ?? 0
|
||||
},
|
||||
tier3: {
|
||||
users: tier3LimitSet[LimitId.USERS]?.value ?? 0,
|
||||
sites: tier3LimitSet[LimitId.SITES]?.value ?? 0,
|
||||
domains: tier3LimitSet[LimitId.DOMAINS]?.value ?? 0,
|
||||
remoteNodes: tier3LimitSet[LimitId.REMOTE_EXIT_NODES]?.value ?? 0,
|
||||
organizations: tier3LimitSet[LimitId.ORGANIZATIONS]?.value ?? 0,
|
||||
publicResources: tier3LimitSet[LimitId.PUBLIC_RESOURCES]?.value ?? 0,
|
||||
privateResources: tier3LimitSet[LimitId.PRIVATE_RESOURCES]?.value ?? 0,
|
||||
machineClients: tier3LimitSet[LimitId.MACHINE_CLIENTS]?.value ?? 0
|
||||
organizations: tier3LimitSet[LimitId.ORGANIZATIONS]?.value ?? 0
|
||||
},
|
||||
enterprise: {
|
||||
users: 0, // Custom for enterprise
|
||||
sites: 0, // Custom for enterprise
|
||||
domains: 0, // Custom for enterprise
|
||||
remoteNodes: 0, // Custom for enterprise
|
||||
organizations: 0, // Custom for enterprise
|
||||
publicResources: 0, // Custom for enterprise
|
||||
privateResources: 0, // Custom for enterprise
|
||||
machineClients: 0 // Custom for enterprise
|
||||
organizations: 0 // Custom for enterprise
|
||||
}
|
||||
};
|
||||
|
||||
@@ -252,9 +234,6 @@ export default function BillingPage() {
|
||||
const DOMAINS = "domains";
|
||||
const REMOTE_EXIT_NODES = "remoteExitNodes";
|
||||
const ORGINIZATIONS = "organizations";
|
||||
const PUBLIC_RESOURCES = "publicResources";
|
||||
const PRIVATE_RESOURCES = "privateResources";
|
||||
const MACHINE_CLIENTS = "machineClients";
|
||||
|
||||
// Confirmation dialog state
|
||||
const [showConfirmDialog, setShowConfirmDialog] = useState(false);
|
||||
@@ -290,13 +269,7 @@ export default function BillingPage() {
|
||||
setHasSubscription(
|
||||
tierSub.subscription.status === "active"
|
||||
);
|
||||
// expiresAt is only meaningful while the trial hasn't
|
||||
// actually run out yet; a stale row with a past
|
||||
// expiresAt should no longer be treated as a live trial
|
||||
const expiresAt = tierSub.subscription.expiresAt;
|
||||
setIsTrial(
|
||||
expiresAt != null && expiresAt * 1000 > Date.now()
|
||||
);
|
||||
setIsTrial(tierSub.subscription.expiresAt != null);
|
||||
}
|
||||
|
||||
// Find license subscription
|
||||
@@ -824,45 +797,6 @@ export default function BillingPage() {
|
||||
});
|
||||
}
|
||||
|
||||
// Check public resources
|
||||
const publicResourcesUsage = getUsageValue(PUBLIC_RESOURCES);
|
||||
if (
|
||||
limits.publicResources > 0 &&
|
||||
publicResourcesUsage > limits.publicResources
|
||||
) {
|
||||
violations.push({
|
||||
feature: "Public Resources",
|
||||
currentUsage: publicResourcesUsage,
|
||||
newLimit: limits.publicResources
|
||||
});
|
||||
}
|
||||
|
||||
// Check private resources
|
||||
const privateResourcesUsage = getUsageValue(PRIVATE_RESOURCES);
|
||||
if (
|
||||
limits.privateResources > 0 &&
|
||||
privateResourcesUsage > limits.privateResources
|
||||
) {
|
||||
violations.push({
|
||||
feature: "Private Resources",
|
||||
currentUsage: privateResourcesUsage,
|
||||
newLimit: limits.privateResources
|
||||
});
|
||||
}
|
||||
|
||||
// Check machine clients
|
||||
const machineClientsUsage = getUsageValue(MACHINE_CLIENTS);
|
||||
if (
|
||||
limits.machineClients > 0 &&
|
||||
machineClientsUsage > limits.machineClients
|
||||
) {
|
||||
violations.push({
|
||||
feature: "Machine Clients",
|
||||
currentUsage: machineClientsUsage,
|
||||
newLimit: limits.machineClients
|
||||
});
|
||||
}
|
||||
|
||||
return violations;
|
||||
};
|
||||
|
||||
@@ -1063,23 +997,21 @@ export default function BillingPage() {
|
||||
</SettingsSectionDescription>
|
||||
</SettingsSectionHeader>
|
||||
<SettingsSectionBody>
|
||||
<div className="grid grid-cols-1 md:grid-cols-4 gap-6">
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
{/* Current Usage */}
|
||||
<div className="border rounded-lg p-4 md:col-span-1">
|
||||
<div className="border rounded-lg p-4">
|
||||
<div className="text-sm text-muted-foreground mb-2">
|
||||
{t("billingCurrentUsage") || "Current Usage"}
|
||||
</div>
|
||||
<div className="flex flex-col items-start gap-1">
|
||||
<div className="flex items-baseline gap-2">
|
||||
<span className="text-3xl font-semibold">
|
||||
{getUserCount()}
|
||||
</span>
|
||||
<span className="text-lg">
|
||||
{t("billingUsers") || "Users"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-baseline gap-2">
|
||||
<span className="text-3xl font-semibold">
|
||||
{getUserCount()}
|
||||
</span>
|
||||
<span className="text-lg">
|
||||
{t("billingUsers") || "Users"}
|
||||
</span>
|
||||
{hasSubscription && getPricePerUser() > 0 && (
|
||||
<div className="text-sm text-muted-foreground">
|
||||
<div className="text-sm text-muted-foreground mt-1">
|
||||
x ${getPricePerUser()} / month = $
|
||||
{getUserCount() * getPricePerUser()} /
|
||||
month
|
||||
@@ -1089,11 +1021,11 @@ export default function BillingPage() {
|
||||
</div>
|
||||
|
||||
{/* Maximum Limits */}
|
||||
<div className="border rounded-lg p-4 md:col-span-3">
|
||||
<div className="border rounded-lg p-4">
|
||||
<div className="text-sm text-muted-foreground mb-3">
|
||||
{t("billingMaximumLimits") || "Maximum Limits"}
|
||||
</div>
|
||||
<InfoSections cols={8}>
|
||||
<InfoSections cols={5}>
|
||||
<InfoSection>
|
||||
<InfoSectionTitle className="flex items-center gap-1 text-xs">
|
||||
{t("billingUsers") || "Users"}
|
||||
@@ -1376,168 +1308,6 @@ export default function BillingPage() {
|
||||
)}
|
||||
</InfoSectionContent>
|
||||
</InfoSection>
|
||||
<InfoSection>
|
||||
<InfoSectionTitle className="flex items-center gap-1 text-xs">
|
||||
{t("billingPublicResources") ||
|
||||
"Public Resources"}
|
||||
</InfoSectionTitle>
|
||||
<InfoSectionContent className="text-sm">
|
||||
{isOverLimit(PUBLIC_RESOURCES) ? (
|
||||
<Tooltip>
|
||||
<TooltipTrigger className="flex items-center gap-1">
|
||||
<AlertTriangle className="h-3 w-3 text-orange-400" />
|
||||
<span
|
||||
className={cn(
|
||||
"text-orange-600 dark:text-orange-400 font-medium"
|
||||
)}
|
||||
>
|
||||
{getLimitValue(
|
||||
PUBLIC_RESOURCES
|
||||
) ??
|
||||
t(
|
||||
"billingUnlimited"
|
||||
) ??
|
||||
"∞"}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>
|
||||
{t(
|
||||
"billingUsageExceedsLimit",
|
||||
{
|
||||
current:
|
||||
getUsageValue(
|
||||
PUBLIC_RESOURCES
|
||||
),
|
||||
limit:
|
||||
getLimitValue(
|
||||
PUBLIC_RESOURCES
|
||||
) ?? 0
|
||||
}
|
||||
) ||
|
||||
`Current usage (${getUsageValue(PUBLIC_RESOURCES)}) exceeds limit (${getLimitValue(PUBLIC_RESOURCES)})`}
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<>
|
||||
{getLimitValue(
|
||||
PUBLIC_RESOURCES
|
||||
) ??
|
||||
t("billingUnlimited") ??
|
||||
"∞"}
|
||||
</>
|
||||
)}
|
||||
</InfoSectionContent>
|
||||
</InfoSection>
|
||||
<InfoSection>
|
||||
<InfoSectionTitle className="flex items-center gap-1 text-xs">
|
||||
{t("billingPrivateResources") ||
|
||||
"Private Resources"}
|
||||
</InfoSectionTitle>
|
||||
<InfoSectionContent className="text-sm">
|
||||
{isOverLimit(PRIVATE_RESOURCES) ? (
|
||||
<Tooltip>
|
||||
<TooltipTrigger className="flex items-center gap-1">
|
||||
<AlertTriangle className="h-3 w-3 text-orange-400" />
|
||||
<span
|
||||
className={cn(
|
||||
"text-orange-600 dark:text-orange-400 font-medium"
|
||||
)}
|
||||
>
|
||||
{getLimitValue(
|
||||
PRIVATE_RESOURCES
|
||||
) ??
|
||||
t(
|
||||
"billingUnlimited"
|
||||
) ??
|
||||
"∞"}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>
|
||||
{t(
|
||||
"billingUsageExceedsLimit",
|
||||
{
|
||||
current:
|
||||
getUsageValue(
|
||||
PRIVATE_RESOURCES
|
||||
),
|
||||
limit:
|
||||
getLimitValue(
|
||||
PRIVATE_RESOURCES
|
||||
) ?? 0
|
||||
}
|
||||
) ||
|
||||
`Current usage (${getUsageValue(PRIVATE_RESOURCES)}) exceeds limit (${getLimitValue(PRIVATE_RESOURCES)})`}
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<>
|
||||
{getLimitValue(
|
||||
PRIVATE_RESOURCES
|
||||
) ??
|
||||
t("billingUnlimited") ??
|
||||
"∞"}
|
||||
</>
|
||||
)}
|
||||
</InfoSectionContent>
|
||||
</InfoSection>
|
||||
<InfoSection>
|
||||
<InfoSectionTitle className="flex items-center gap-1 text-xs">
|
||||
{t("billingMachineClients") ||
|
||||
"Machine Clients"}
|
||||
</InfoSectionTitle>
|
||||
<InfoSectionContent className="text-sm">
|
||||
{isOverLimit(MACHINE_CLIENTS) ? (
|
||||
<Tooltip>
|
||||
<TooltipTrigger className="flex items-center gap-1">
|
||||
<AlertTriangle className="h-3 w-3 text-orange-400" />
|
||||
<span
|
||||
className={cn(
|
||||
"text-orange-600 dark:text-orange-400 font-medium"
|
||||
)}
|
||||
>
|
||||
{getLimitValue(
|
||||
MACHINE_CLIENTS
|
||||
) ??
|
||||
t(
|
||||
"billingUnlimited"
|
||||
) ??
|
||||
"∞"}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>
|
||||
{t(
|
||||
"billingUsageExceedsLimit",
|
||||
{
|
||||
current:
|
||||
getUsageValue(
|
||||
MACHINE_CLIENTS
|
||||
),
|
||||
limit:
|
||||
getLimitValue(
|
||||
MACHINE_CLIENTS
|
||||
) ?? 0
|
||||
}
|
||||
) ||
|
||||
`Current usage (${getUsageValue(MACHINE_CLIENTS)}) exceeds limit (${getLimitValue(MACHINE_CLIENTS)})`}
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<>
|
||||
{getLimitValue(
|
||||
MACHINE_CLIENTS
|
||||
) ??
|
||||
t("billingUnlimited") ??
|
||||
"∞"}
|
||||
</>
|
||||
)}
|
||||
</InfoSectionContent>
|
||||
</InfoSection>
|
||||
</InfoSections>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1737,45 +1507,6 @@ export default function BillingPage() {
|
||||
"Remote Nodes"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-muted-foreground/50 shrink-0" />
|
||||
<span>
|
||||
{
|
||||
tierLimits[
|
||||
pendingTier.tier
|
||||
].publicResources
|
||||
}{" "}
|
||||
{t(
|
||||
"billingPublicResources"
|
||||
) || "Public Resources"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-muted-foreground/50 shrink-0" />
|
||||
<span>
|
||||
{
|
||||
tierLimits[
|
||||
pendingTier.tier
|
||||
].privateResources
|
||||
}{" "}
|
||||
{t(
|
||||
"billingPrivateResources"
|
||||
) || "Private Resources"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-muted-foreground/50 shrink-0" />
|
||||
<span>
|
||||
{
|
||||
tierLimits[
|
||||
pendingTier.tier
|
||||
].machineClients
|
||||
}{" "}
|
||||
{t(
|
||||
"billingMachineClients"
|
||||
) || "Machine Clients"}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user