diff --git a/.github/workflows/mirror-node.yml b/.github/workflows/mirror-node.yml new file mode 100644 index 000000000..573d0afe4 --- /dev/null +++ b/.github/workflows/mirror-node.yml @@ -0,0 +1,138 @@ +name: Mirror Node Base Image (Docker Hub to GHCR) + +# Keeps a copy of the official node images our Dockerfiles build FROM in GHCR. +# Anonymous pulls from Docker Hub / ECR Public are rate limited per source IP, +# and GitHub-hosted runners share IPs, so CI builds were failing with 429s. +# Public GHCR packages pull without credentials, so PR builds (including from +# forks) need no secrets. +# +# The tags to mirror are read from the Dockerfiles themselves: any reference to +# ghcr.io//node: is copied from docker.io/library/node:. Bumping +# the Node version in a Dockerfile and pushing it mirrors the new tag +# automatically. The daily run picks up upstream rebuilds (OS security patches) +# of the same tag. +# +# NOTE: the first push creates the ghcr.io//node package as private. +# Make it public once in the org's package settings so builds can pull it +# anonymously. + +permissions: + contents: read + packages: write + +on: + schedule: + - cron: "17 4 * * *" + push: + paths: + - "Dockerfile*" + - ".github/workflows/mirror-node.yml" + workflow_dispatch: + inputs: + tags: + description: "Extra node tags to mirror, space separated (e.g. 24.23.0-slim)" + required: false + default: "" + +concurrency: + group: mirror-node + cancel-in-progress: false + +env: + SOURCE_IMAGE: docker.io/library/node + DEST_IMAGE: ghcr.io/${{ github.repository_owner }}/node + +jobs: + mirror: + # Don't run (and push to their own GHCR) in forks + if: github.repository_owner == 'fosrl' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install skopeo + run: | + sudo apt-get update -y + sudo apt-get install -y skopeo + skopeo --version + + - name: Log in to registries + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + run: | + set -euo pipefail + # runner.temp isn't available in job-level env, so export it here + # for this and later steps. + export REGISTRY_AUTH_FILE="${RUNNER_TEMP}/containers/auth.json" + echo "REGISTRY_AUTH_FILE=${REGISTRY_AUTH_FILE}" >> "$GITHUB_ENV" + mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")" + echo "$GHCR_TOKEN" | skopeo login ghcr.io -u "${{ github.actor }}" --password-stdin + # Authenticated Docker Hub pulls avoid the shared-IP anonymous limit. + # This only runs on trusted refs (schedule/push/dispatch), never PRs. + if [ -n "$DOCKER_HUB_USERNAME" ] && [ -n "$DOCKER_HUB_ACCESS_TOKEN" ]; then + echo "$DOCKER_HUB_ACCESS_TOKEN" | skopeo login docker.io -u "$DOCKER_HUB_USERNAME" --password-stdin + fi + + - name: Collect tags + env: + EXTRA_TAGS: ${{ inputs.tags }} + run: | + set -euo pipefail + { + grep -hoE "${DEST_IMAGE}:[A-Za-z0-9._-]+" Dockerfile* | sed "s|^${DEST_IMAGE}:||" || true + for t in ${EXTRA_TAGS:-}; do echo "$t"; done + } | sort -u > tags.txt + if [ ! -s tags.txt ]; then + echo "::error ::No ${DEST_IMAGE}: references found in Dockerfiles" + exit 1 + fi + echo "Tags to mirror:" + cat tags.txt + + - name: Mirror tags + run: | + set -euo pipefail + errs=0 + while read -r tag; do + if ! [[ "$tag" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "::warning ::Skipping invalid tag '${tag}'" + errs=$((errs+1)); continue + fi + + src="${SOURCE_IMAGE}:${tag}" + dst="${DEST_IMAGE}:${tag}" + + if ! skopeo inspect --raw --retry-times 3 "docker://${src}" > src.json; then + echo "::warning title=Inspect failed::${src}" + errs=$((errs+1)); continue + fi + src_digest="$(skopeo manifest-digest src.json)" + + dst_digest="" + if skopeo inspect --raw --retry-times 3 "docker://${dst}" > dst.json 2>/dev/null; then + dst_digest="$(skopeo manifest-digest dst.json)" + fi + + if [ "$src_digest" = "$dst_digest" ]; then + echo "Up to date: ${dst} (${src_digest})" + continue + fi + + echo "==> Copy ${src} (${src_digest}) -> ${dst}" + # --all keeps every architecture; --preserve-digests keeps the + # mirror byte-identical so it can be pinned by the upstream digest. + if ! skopeo copy --all --preserve-digests --retry-times 3 \ + "docker://${src}" "docker://${dst}"; then + echo "::warning title=Copy failed::${src}" + errs=$((errs+1)) + fi + done < tags.txt + + if [ "$errs" -gt 0 ]; then + echo "::error ::${errs} tag(s) failed to mirror" + exit 1 + fi diff --git a/Dockerfile b/Dockerfile index aff11ae4e..01406a829 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ -# FROM node:24.22.1-slim AS base -FROM public.ecr.aws/docker/library/node:24.22.1-slim AS base +# Mirror of docker.io/library/node, kept in sync by +# .github/workflows/mirror-node.yml (avoids Docker Hub / ECR pull rate limits) +ARG NODE_IMAGE=ghcr.io/fosrl/node:24.22.1-slim + +FROM ${NODE_IMAGE} AS base WORKDIR /app @@ -35,8 +38,7 @@ RUN npm prune --omit=dev && \ # Real files should be present for saas builds, placeholders for oss builds RUN touch /app/GeoLite2-Country.mmdb /app/GeoLite2-ASN.mmdb -# FROM node:24.22.1-slim AS runner -FROM public.ecr.aws/docker/library/node:24.22.1-slim AS runner +FROM ${NODE_IMAGE} AS runner WORKDIR /app diff --git a/Dockerfile.dev b/Dockerfile.dev index 7874b6c55..2a68e578a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,6 @@ -FROM node:24.22.1-alpine +# Mirror of docker.io/library/node, kept in sync by +# .github/workflows/mirror-node.yml +FROM ghcr.io/fosrl/node:24.22.1-alpine WORKDIR /app diff --git a/config/traefik/traefik_config.yml b/config/traefik/traefik_config.yml index 6d98463e3..71cb6179c 100644 --- a/config/traefik/traefik_config.yml +++ b/config/traefik/traefik_config.yml @@ -11,7 +11,7 @@ experimental: plugins: badger: moduleName: github.com/fosrl/badger - version: v1.8.0 + version: v1.7.0 log: level: INFO format: common diff --git a/install/config/docker-compose.yml b/install/config/docker-compose.yml index 05b2fbe8c..9ec616b82 100644 --- a/install/config/docker-compose.yml +++ b/install/config/docker-compose.yml @@ -50,7 +50,7 @@ services: - 80:80{{end}} traefik: - image: docker.io/traefik:v3.7 + image: docker.io/traefik:v3.7.16 container_name: traefik restart: unless-stopped {{if .InstallGerbil}}network_mode: service:gerbil # Ports appear on the gerbil service{{end}}{{if not .InstallGerbil}} diff --git a/server/lib/statusHistory.test.ts b/server/lib/statusHistory.test.ts new file mode 100644 index 000000000..1a35948a2 --- /dev/null +++ b/server/lib/statusHistory.test.ts @@ -0,0 +1,58 @@ +import { computeBuckets } from "./statusHistory"; +import { assertEquals } from "@test/assert"; + +function event(id: number, timestamp: number, status: string) { + return { + entityType: "resource", + entityId: 1, + orgId: "org", + status, + timestamp, + id + }; +} + +// Regression test for https://github.com/fosrl/pangolin/issues/3994 +function testSameSecondEventsUseInsertionOrder() { + console.log("Running computeBuckets same-second ordering tests..."); + + // Two days ago the resource went unhealthy and recovered within the same + // second. The database may return the two rows in either order. + const nowSec = Math.floor(Date.now() / 1000); + const flap = nowSec - 2 * 86400; + const unhealthy = event(1, flap, "unhealthy"); + const healthy = event(2, flap, "healthy"); + + for (const events of [ + [unhealthy, healthy], + [healthy, unhealthy] + ]) { + const { buckets, totalDowntime } = computeBuckets( + events, + 5, + "healthy", + 0 + ); + const order = events.map((e) => e.id).join(","); + assertEquals( + totalDowntime, + 0, + `No downtime when the later event is healthy (rows ${order})` + ); + assertEquals( + buckets.at(-1)!.status, + "good", + `Today should be good (rows ${order})` + ); + } + + console.log("All computeBuckets same-second ordering tests passed!"); +} + +try { + testSameSecondEventsUseInsertionOrder(); + console.log("All tests passed successfully!"); +} catch (error) { + console.error("Test failed:", error); + process.exit(1); +} diff --git a/server/lib/statusHistory.ts b/server/lib/statusHistory.ts index 0ae666dd5..1bcf29dee 100644 --- a/server/lib/statusHistory.ts +++ b/server/lib/statusHistory.ts @@ -56,7 +56,7 @@ export async function getCachedStatusHistory( gte(statusHistory.timestamp, startSec) ) ) - .orderBy(asc(statusHistory.timestamp)); + .orderBy(asc(statusHistory.timestamp), asc(statusHistory.id)); // Fetch the last known state before the window so that entities that // haven't changed status recently still show the correct status rather @@ -71,7 +71,7 @@ export async function getCachedStatusHistory( lt(statusHistory.timestamp, startSec) ) ) - .orderBy(desc(statusHistory.timestamp)) + .orderBy(desc(statusHistory.timestamp), desc(statusHistory.id)) .limit(1); const priorStatus = lastKnownEvent?.status ?? null; @@ -168,6 +168,14 @@ export function computeBuckets( // don't drift as the cache expires and is recomputed const todayMidnightSec = localMidnightSec(tzOffsetMinutes); + // Several status changes can land in the same second (e.g. a resource + // going degraded -> healthy as each target reports in), and the database + // does not keep a stable order for rows with equal timestamps. Order by id + // as well so the last event of the second is the one that was written last. + events = [...events].sort( + (a, b) => a.timestamp - b.timestamp || a.id - b.id + ); + const buckets: StatusHistoryDayBucket[] = []; let totalDowntime = 0; @@ -328,7 +336,7 @@ export async function getBatchedStatusHistory( gte(statusHistory.timestamp, startSec) ) ) - .orderBy(asc(statusHistory.timestamp)); + .orderBy(asc(statusHistory.timestamp), asc(statusHistory.id)); // Fetch the last known state before the window so that entities that // haven't changed status recently still show the correct status rather @@ -349,7 +357,7 @@ export async function getBatchedStatusHistory( status: statusHistory.status, timestamp: statusHistory.timestamp, row_number: - sql`ROW_NUMBER() OVER (PARTITION BY ${statusHistory.entityId} ORDER BY ${statusHistory.timestamp} DESC)`.as( + sql`ROW_NUMBER() OVER (PARTITION BY ${statusHistory.entityId} ORDER BY ${statusHistory.timestamp} DESC, ${statusHistory.id} DESC)`.as( "row_number" ) })