From 05d28748449917eb17d3c05461d905c96969edf3 Mon Sep 17 00:00:00 2001 From: RishavTiwari25 Date: Mon, 26 Oct 2026 13:56:19 +0530 Subject: [PATCH 1/5] fix(status-history): order same-second events by id Status rows written in the same second have equal timestamps, and the batched query returns them in an order that depends on which other resources are in the request. When a recovery event sorted before the failure it followed, the resource stayed unhealthy until its next event. Break timestamp ties by id in the queries and in computeBuckets. Fixes #3994 --- server/lib/statusHistory.test.ts | 58 ++++++++++++++++++++++++++++++++ server/lib/statusHistory.ts | 16 ++++++--- 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 server/lib/statusHistory.test.ts diff --git a/server/lib/statusHistory.test.ts b/server/lib/statusHistory.test.ts new file mode 100644 index 000000000..1a35948a2 --- /dev/null +++ b/server/lib/statusHistory.test.ts @@ -0,0 +1,58 @@ +import { computeBuckets } from "./statusHistory"; +import { assertEquals } from "@test/assert"; + +function event(id: number, timestamp: number, status: string) { + return { + entityType: "resource", + entityId: 1, + orgId: "org", + status, + timestamp, + id + }; +} + +// Regression test for https://github.com/fosrl/pangolin/issues/3994 +function testSameSecondEventsUseInsertionOrder() { + console.log("Running computeBuckets same-second ordering tests..."); + + // Two days ago the resource went unhealthy and recovered within the same + // second. The database may return the two rows in either order. + const nowSec = Math.floor(Date.now() / 1000); + const flap = nowSec - 2 * 86400; + const unhealthy = event(1, flap, "unhealthy"); + const healthy = event(2, flap, "healthy"); + + for (const events of [ + [unhealthy, healthy], + [healthy, unhealthy] + ]) { + const { buckets, totalDowntime } = computeBuckets( + events, + 5, + "healthy", + 0 + ); + const order = events.map((e) => e.id).join(","); + assertEquals( + totalDowntime, + 0, + `No downtime when the later event is healthy (rows ${order})` + ); + assertEquals( + buckets.at(-1)!.status, + "good", + `Today should be good (rows ${order})` + ); + } + + console.log("All computeBuckets same-second ordering tests passed!"); +} + +try { + testSameSecondEventsUseInsertionOrder(); + console.log("All tests passed successfully!"); +} catch (error) { + console.error("Test failed:", error); + process.exit(1); +} diff --git a/server/lib/statusHistory.ts b/server/lib/statusHistory.ts index 0ae666dd5..1bcf29dee 100644 --- a/server/lib/statusHistory.ts +++ b/server/lib/statusHistory.ts @@ -56,7 +56,7 @@ export async function getCachedStatusHistory( gte(statusHistory.timestamp, startSec) ) ) - .orderBy(asc(statusHistory.timestamp)); + .orderBy(asc(statusHistory.timestamp), asc(statusHistory.id)); // Fetch the last known state before the window so that entities that // haven't changed status recently still show the correct status rather @@ -71,7 +71,7 @@ export async function getCachedStatusHistory( lt(statusHistory.timestamp, startSec) ) ) - .orderBy(desc(statusHistory.timestamp)) + .orderBy(desc(statusHistory.timestamp), desc(statusHistory.id)) .limit(1); const priorStatus = lastKnownEvent?.status ?? null; @@ -168,6 +168,14 @@ export function computeBuckets( // don't drift as the cache expires and is recomputed const todayMidnightSec = localMidnightSec(tzOffsetMinutes); + // Several status changes can land in the same second (e.g. a resource + // going degraded -> healthy as each target reports in), and the database + // does not keep a stable order for rows with equal timestamps. Order by id + // as well so the last event of the second is the one that was written last. + events = [...events].sort( + (a, b) => a.timestamp - b.timestamp || a.id - b.id + ); + const buckets: StatusHistoryDayBucket[] = []; let totalDowntime = 0; @@ -328,7 +336,7 @@ export async function getBatchedStatusHistory( gte(statusHistory.timestamp, startSec) ) ) - .orderBy(asc(statusHistory.timestamp)); + .orderBy(asc(statusHistory.timestamp), asc(statusHistory.id)); // Fetch the last known state before the window so that entities that // haven't changed status recently still show the correct status rather @@ -349,7 +357,7 @@ export async function getBatchedStatusHistory( status: statusHistory.status, timestamp: statusHistory.timestamp, row_number: - sql`ROW_NUMBER() OVER (PARTITION BY ${statusHistory.entityId} ORDER BY ${statusHistory.timestamp} DESC)`.as( + sql`ROW_NUMBER() OVER (PARTITION BY ${statusHistory.entityId} ORDER BY ${statusHistory.timestamp} DESC, ${statusHistory.id} DESC)`.as( "row_number" ) }) From fbb0b4b6d2462aa037d67b10c77c9f770d9fc353 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 26 Oct 2026 11:21:50 -0400 Subject: [PATCH 2/5] Push and pull from ghcr --- .github/workflows/mirror-node.yml | 136 ++++++++++++++++++++++++++++++ Dockerfile | 10 ++- Dockerfile.dev | 4 +- 3 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/mirror-node.yml diff --git a/.github/workflows/mirror-node.yml b/.github/workflows/mirror-node.yml new file mode 100644 index 000000000..879f4957d --- /dev/null +++ b/.github/workflows/mirror-node.yml @@ -0,0 +1,136 @@ +name: Mirror Node Base Image (Docker Hub to GHCR) + +# Keeps a copy of the official node images our Dockerfiles build FROM in GHCR. +# Anonymous pulls from Docker Hub / ECR Public are rate limited per source IP, +# and GitHub-hosted runners share IPs, so CI builds were failing with 429s. +# Public GHCR packages pull without credentials, so PR builds (including from +# forks) need no secrets. +# +# The tags to mirror are read from the Dockerfiles themselves: any reference to +# ghcr.io//node: is copied from docker.io/library/node:. Bumping +# the Node version in a Dockerfile and pushing it mirrors the new tag +# automatically. The daily run picks up upstream rebuilds (OS security patches) +# of the same tag. +# +# NOTE: the first push creates the ghcr.io//node package as private. +# Make it public once in the org's package settings so builds can pull it +# anonymously. + +permissions: + contents: read + packages: write + +on: + schedule: + - cron: "17 4 * * *" + push: + paths: + - "Dockerfile*" + - ".github/workflows/mirror-node.yml" + workflow_dispatch: + inputs: + tags: + description: "Extra node tags to mirror, space separated (e.g. 24.23.0-slim)" + required: false + default: "" + +concurrency: + group: mirror-node + cancel-in-progress: false + +env: + SOURCE_IMAGE: docker.io/library/node + DEST_IMAGE: ghcr.io/${{ github.repository_owner }}/node + +jobs: + mirror: + # Don't run (and push to their own GHCR) in forks + if: github.repository_owner == 'fosrl' + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install skopeo + run: | + sudo apt-get update -y + sudo apt-get install -y skopeo + skopeo --version + + - name: Log in to registries + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + run: | + set -euo pipefail + mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" + echo "$GHCR_TOKEN" | skopeo login ghcr.io -u "${{ github.actor }}" --password-stdin + # Authenticated Docker Hub pulls avoid the shared-IP anonymous limit. + # This only runs on trusted refs (schedule/push/dispatch), never PRs. + if [ -n "$DOCKER_HUB_USERNAME" ] && [ -n "$DOCKER_HUB_ACCESS_TOKEN" ]; then + echo "$DOCKER_HUB_ACCESS_TOKEN" | skopeo login docker.io -u "$DOCKER_HUB_USERNAME" --password-stdin + fi + + - name: Collect tags + env: + EXTRA_TAGS: ${{ inputs.tags }} + run: | + set -euo pipefail + { + grep -hoE "${DEST_IMAGE}:[A-Za-z0-9._-]+" Dockerfile* | sed "s|^${DEST_IMAGE}:||" || true + for t in ${EXTRA_TAGS:-}; do echo "$t"; done + } | sort -u > tags.txt + if [ ! -s tags.txt ]; then + echo "::error ::No ${DEST_IMAGE}: references found in Dockerfiles" + exit 1 + fi + echo "Tags to mirror:" + cat tags.txt + + - name: Mirror tags + run: | + set -euo pipefail + errs=0 + while read -r tag; do + if ! [[ "$tag" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "::warning ::Skipping invalid tag '${tag}'" + errs=$((errs+1)); continue + fi + + src="${SOURCE_IMAGE}:${tag}" + dst="${DEST_IMAGE}:${tag}" + + if ! skopeo inspect --raw --retry-times 3 "docker://${src}" > src.json; then + echo "::warning title=Inspect failed::${src}" + errs=$((errs+1)); continue + fi + src_digest="$(skopeo manifest-digest src.json)" + + dst_digest="" + if skopeo inspect --raw --retry-times 3 "docker://${dst}" > dst.json 2>/dev/null; then + dst_digest="$(skopeo manifest-digest dst.json)" + fi + + if [ "$src_digest" = "$dst_digest" ]; then + echo "Up to date: ${dst} (${src_digest})" + continue + fi + + echo "==> Copy ${src} (${src_digest}) -> ${dst}" + # --all keeps every architecture; --preserve-digests keeps the + # mirror byte-identical so it can be pinned by the upstream digest. + if ! skopeo copy --all --preserve-digests --retry-times 3 \ + "docker://${src}" "docker://${dst}"; then + echo "::warning title=Copy failed::${src}" + errs=$((errs+1)) + fi + done < tags.txt + + if [ "$errs" -gt 0 ]; then + echo "::error ::${errs} tag(s) failed to mirror" + exit 1 + fi diff --git a/Dockerfile b/Dockerfile index aff11ae4e..01406a829 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ -# FROM node:24.22.1-slim AS base -FROM public.ecr.aws/docker/library/node:24.22.1-slim AS base +# Mirror of docker.io/library/node, kept in sync by +# .github/workflows/mirror-node.yml (avoids Docker Hub / ECR pull rate limits) +ARG NODE_IMAGE=ghcr.io/fosrl/node:24.22.1-slim + +FROM ${NODE_IMAGE} AS base WORKDIR /app @@ -35,8 +38,7 @@ RUN npm prune --omit=dev && \ # Real files should be present for saas builds, placeholders for oss builds RUN touch /app/GeoLite2-Country.mmdb /app/GeoLite2-ASN.mmdb -# FROM node:24.22.1-slim AS runner -FROM public.ecr.aws/docker/library/node:24.22.1-slim AS runner +FROM ${NODE_IMAGE} AS runner WORKDIR /app diff --git a/Dockerfile.dev b/Dockerfile.dev index 7874b6c55..2a68e578a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,6 @@ -FROM node:24.22.1-alpine +# Mirror of docker.io/library/node, kept in sync by +# .github/workflows/mirror-node.yml +FROM ghcr.io/fosrl/node:24.22.1-alpine WORKDIR /app From 14d337b86a489439f00e5667f7f7ddbf44d63aeb Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 26 Oct 2026 11:24:06 -0400 Subject: [PATCH 3/5] Fix mirror error --- .github/workflows/mirror-node.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mirror-node.yml b/.github/workflows/mirror-node.yml index 879f4957d..573d0afe4 100644 --- a/.github/workflows/mirror-node.yml +++ b/.github/workflows/mirror-node.yml @@ -48,8 +48,6 @@ jobs: if: github.repository_owner == 'fosrl' runs-on: ubuntu-latest timeout-minutes: 30 - env: - REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -67,6 +65,10 @@ jobs: DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} run: | set -euo pipefail + # runner.temp isn't available in job-level env, so export it here + # for this and later steps. + export REGISTRY_AUTH_FILE="${RUNNER_TEMP}/containers/auth.json" + echo "REGISTRY_AUTH_FILE=${REGISTRY_AUTH_FILE}" >> "$GITHUB_ENV" mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" echo "$GHCR_TOKEN" | skopeo login ghcr.io -u "${{ github.actor }}" --password-stdin # Authenticated Docker Hub pulls avoid the shared-IP anonymous limit. From 79acfb6c17a8d06fa15921b5f49fd0d3c605f214 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 26 Oct 2026 11:56:56 -0400 Subject: [PATCH 4/5] Pin traefik more specific --- install/config/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/config/docker-compose.yml b/install/config/docker-compose.yml index 05b2fbe8c..9ec616b82 100644 --- a/install/config/docker-compose.yml +++ b/install/config/docker-compose.yml @@ -50,7 +50,7 @@ services: - 80:80{{end}} traefik: - image: docker.io/traefik:v3.7 + image: docker.io/traefik:v3.7.16 container_name: traefik restart: unless-stopped {{if .InstallGerbil}}network_mode: service:gerbil # Ports appear on the gerbil service{{end}}{{if not .InstallGerbil}} From 5c517e1841d66444a41aa5990203966a31e3b6b4 Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 26 Oct 2026 11:57:59 -0400 Subject: [PATCH 5/5] Revert to working version --- config/traefik/traefik_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/traefik/traefik_config.yml b/config/traefik/traefik_config.yml index 6d98463e3..71cb6179c 100644 --- a/config/traefik/traefik_config.yml +++ b/config/traefik/traefik_config.yml @@ -11,7 +11,7 @@ experimental: plugins: badger: moduleName: github.com/fosrl/badger - version: v1.8.0 + version: v1.7.0 log: level: INFO format: common