pangolin_mirror/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-02-25 22:36:38 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
c4dcec463abc10074c37563216ed8e62e51e43b3
pangolin/.github/workflows/cicd.yml
Laurence 9460e28c7b ehance(installer): use ldflags to inject versions 
Instead of the CI/CD using sed to replace the 'replaceme' text we can instead use ldflags which can inject variables at build time to the versions. The makefile had a bunch of workarounds for dev so these have been removed to cleanup etc etc and fetchs versions from the gh api directly if the variables are not injected like the CI/CD does
2026-02-18 09:43:41 +00:00

587 lines
25 KiB
YAML
Raw Blame History

name: Public CICD Pipeline
# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries.
# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events.
permissions:
contents: read
packages: write # for GHCR push
id-token: write # for Cosign Keyless (OIDC) Signing
# Required secrets:
# - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN: push to Docker Hub
# - GITHUB_TOKEN: used for GHCR login and OIDC keyless signing
# - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY: for key-based signing
on:
push:
tags:
- "[0-9]+.[0-9]+.[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
pre-run:
runs-on: ubuntu-latest
permissions: write-all
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v5
with:
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
role-duration-seconds: 3600
aws-region: ${{ secrets.AWS_REGION }}
- name: Verify AWS identity
run: aws sts get-caller-identity
- name: Start EC2 instances
run: |
aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
echo "EC2 instances started"
release-arm:
name: Build and Release (ARM64)
runs-on: [self-hosted, linux, arm64, us-east-1]
needs: [pre-run]
if: >-
${{
needs.pre-run.result == 'success'
}}
# Job-level timeout to avoid runaway or stuck runs
timeout-minutes: 120
env:
# Target images
DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
steps:
- name: Checkout code
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Monitor storage space
run: |
THRESHOLD=75
USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
echo "Used space: $USED_SPACE%"
if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
echo "Used space is below the threshold of 75% free. Running Docker system prune."
echo y | docker system prune -a
else
echo "Storage space is above the threshold. No action needed."
fi
- name: Log in to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: docker.io
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Extract tag name
id: get-tag
run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
shell: bash
- name: Update version in package.json
run: |
TAG=${{ env.TAG }}
sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
cat server/lib/consts.ts
shell: bash
- name: Check if release candidate
id: check-rc
run: |
TAG=${{ env.TAG }}
if [[ "$TAG" == *"-rc."* ]]; then
echo "IS_RC=true" >> $GITHUB_ENV
else
echo "IS_RC=false" >> $GITHUB_ENV
fi
shell: bash
- name: Build and push Docker images (Docker Hub - ARM64)
run: |
TAG=${{ env.TAG }}
if [ "$IS_RC" = "true" ]; then
make build-rc-arm tag=$TAG
else
make build-release-arm tag=$TAG
fi
echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
shell: bash
release-amd:
name: Build and Release (AMD64)
runs-on: [self-hosted, linux, x64, us-east-1]
needs: [pre-run]
if: >-
${{
needs.pre-run.result == 'success'
}}
# Job-level timeout to avoid runaway or stuck runs
timeout-minutes: 120
env:
# Target images
DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
steps:
- name: Checkout code
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Monitor storage space
run: |
THRESHOLD=75
USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
echo "Used space: $USED_SPACE%"
if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
echo "Used space is below the threshold of 75% free. Running Docker system prune."
echo y | docker system prune -a
else
echo "Storage space is above the threshold. No action needed."
fi
- name: Log in to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: docker.io
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Extract tag name
id: get-tag
run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
shell: bash
- name: Update version in package.json
run: |
TAG=${{ env.TAG }}
sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
cat server/lib/consts.ts
shell: bash
- name: Check if release candidate
id: check-rc
run: |
TAG=${{ env.TAG }}
if [[ "$TAG" == *"-rc."* ]]; then
echo "IS_RC=true" >> $GITHUB_ENV
else
echo "IS_RC=false" >> $GITHUB_ENV
fi
shell: bash
- name: Build and push Docker images (Docker Hub - AMD64)
run: |
TAG=${{ env.TAG }}
if [ "$IS_RC" = "true" ]; then
make build-rc-amd tag=$TAG
else
make build-release-amd tag=$TAG
fi
echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
shell: bash
create-manifest:
name: Create Multi-Arch Manifests
runs-on: [self-hosted, linux, x64, us-east-1]
needs: [release-arm, release-amd]
if: >-
${{
needs.release-arm.result == 'success' &&
needs.release-amd.result == 'success'
}}
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Log in to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: docker.io
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Extract tag name
id: get-tag
run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
shell: bash
- name: Check if release candidate
id: check-rc
run: |
TAG=${{ env.TAG }}
if [[ "$TAG" == *"-rc."* ]]; then
echo "IS_RC=true" >> $GITHUB_ENV
else
echo "IS_RC=false" >> $GITHUB_ENV
fi
shell: bash
- name: Create multi-arch manifests
run: |
TAG=${{ env.TAG }}
if [ "$IS_RC" = "true" ]; then
make create-manifests-rc tag=$TAG
else
make create-manifests tag=$TAG
fi
echo "Created multi-arch manifests for tag: ${TAG}"
shell: bash
sign-and-package:
name: Sign and Package
runs-on: [self-hosted, linux, x64, us-east-1]
needs: [release-arm, release-amd, create-manifest]
if: >-
${{
needs.release-arm.result == 'success' &&
needs.release-amd.result == 'success' &&
needs.create-manifest.result == 'success'
}}
# Job-level timeout to avoid runaway or stuck runs
timeout-minutes: 120
env:
# Target images
DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
steps:
- name: Checkout code
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Extract tag name
id: get-tag
run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
shell: bash
- name: Install Go
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
with:
go-version: 1.24
- name: Update version in package.json
run: |
TAG=${{ env.TAG }}
sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
cat server/lib/consts.ts
shell: bash
- name: Pull latest Gerbil version
id: get-gerbil-tag
run: |
LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/gerbil/tags | jq -r '.[0].name')
echo "LATEST_GERBIL_TAG=$LATEST_TAG" >> $GITHUB_ENV
shell: bash
- name: Pull latest Badger version
id: get-badger-tag
run: |
LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/badger/tags | jq -r '.[0].name')
echo "LATEST_BADGER_TAG=$LATEST_TAG" >> $GITHUB_ENV
shell: bash
- name: Build installer
working-directory: install
run: |
make go-build-release \
PANGOLIN_VERSION=${{ env.TAG }} \
GERBIL_VERSION=${{ env.LATEST_GERBIL_TAG }} \
BADGER_VERSION=${{ env.LATEST_BADGER_TAG }}
shell: bash
- name: Upload artifacts from /install/bin
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: install-bin
path: install/bin/
- name: Install skopeo + jq
# skopeo: copy/inspect images between registries
# jq: JSON parsing tool used to extract digest values
run: |
sudo apt-get update -y
sudo apt-get install -y skopeo jq
skopeo --version
shell: bash
- name: Login to GHCR
env:
REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
run: |
mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}"
shell: bash
- name: Copy tags from Docker Hub to GHCR
# Mirror the already-built images (all architectures) to GHCR so we can sign them
# Wait a bit for both architectures to be available in Docker Hub manifest
env:
REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
run: |
set -euo pipefail
TAG=${{ env.TAG }}
MAJOR_TAG=$(echo $TAG | cut -d. -f1)
MINOR_TAG=$(echo $TAG | cut -d. -f1,2)
echo "Waiting for multi-arch manifests to be ready..."
sleep 30
# Determine if this is an RC release
IS_RC="false"
if [[ "$TAG" == *"-rc."* ]]; then
IS_RC="true"
fi
if [ "$IS_RC" = "true" ]; then
echo "RC release detected - copying version-specific tags only"
# SQLite OSS
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:$TAG \
docker://$GHCR_IMAGE:$TAG
# PostgreSQL OSS
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:postgresql-$TAG \
docker://$GHCR_IMAGE:postgresql-$TAG
# SQLite Enterprise
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-${TAG}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:ee-$TAG \
docker://$GHCR_IMAGE:ee-$TAG
# PostgreSQL Enterprise
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG \
docker://$GHCR_IMAGE:ee-postgresql-$TAG
else
echo "Regular release detected - copying all tags (latest, major, minor, full version)"
# SQLite OSS - all tags
for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:${TAG_SUFFIX}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:$TAG_SUFFIX \
docker://$GHCR_IMAGE:$TAG_SUFFIX
done
# PostgreSQL OSS - all tags
for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG_SUFFIX}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:postgresql-$TAG_SUFFIX \
docker://$GHCR_IMAGE:postgresql-$TAG_SUFFIX
done
# SQLite Enterprise - all tags
for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-${TAG_SUFFIX}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:ee-$TAG_SUFFIX \
docker://$GHCR_IMAGE:ee-$TAG_SUFFIX
done
# PostgreSQL Enterprise - all tags
for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG_SUFFIX}"
skopeo copy --all --retry-times 3 \
docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG_SUFFIX \
docker://$GHCR_IMAGE:ee-postgresql-$TAG_SUFFIX
done
fi
echo "All images copied successfully to GHCR!"
shell: bash
- name: Login to GitHub Container Registry (for cosign)
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install cosign
# cosign is used to sign and verify container images (key and keyless)
uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0
- name: Dual-sign and verify (GHCR & Docker Hub)
# Sign each image by digest using keyless (OIDC) and key-based signing,
# then verify both the public key signature and the keyless OIDC signature.
env:
TAG: ${{ env.TAG }}
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
COSIGN_YES: "true"
run: |
set -euo pipefail
issuer="https://token.actions.githubusercontent.com"
id_regex="^https://github.com/${{ github.repository }}/.+" # accept this repo (all workflows/refs)
# Track failures
FAILED_TAGS=()
SUCCESSFUL_TAGS=()
# Determine if this is an RC release
IS_RC="false"
if [[ "$TAG" == *"-rc."* ]]; then
IS_RC="true"
fi
# Define image variants to sign
if [ "$IS_RC" = "true" ]; then
echo "RC release - signing version-specific tags only"
IMAGE_TAGS=(
"${TAG}"
"postgresql-${TAG}"
"ee-${TAG}"
"ee-postgresql-${TAG}"
)
else
echo "Regular release - signing all tags"
MAJOR_TAG=$(echo $TAG | cut -d. -f1)
MINOR_TAG=$(echo $TAG | cut -d. -f1,2)
IMAGE_TAGS=(
"latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"
"postgresql-latest" "postgresql-$MAJOR_TAG" "postgresql-$MINOR_TAG" "postgresql-$TAG"
"ee-latest" "ee-$MAJOR_TAG" "ee-$MINOR_TAG" "ee-$TAG"
"ee-postgresql-latest" "ee-postgresql-$MAJOR_TAG" "ee-postgresql-$MINOR_TAG" "ee-postgresql-$TAG"
)
fi
# Sign each image variant for both registries
for BASE_IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do
for IMAGE_TAG in "${IMAGE_TAGS[@]}"; do
echo "Processing ${BASE_IMAGE}:${IMAGE_TAG}"
TAG_FAILED=false
# Wrap the entire tag processing in error handling
(
set -e
DIGEST="$(skopeo inspect --retry-times 3 docker://${BASE_IMAGE}:${IMAGE_TAG} | jq -r '.Digest')"
REF="${BASE_IMAGE}@${DIGEST}"
echo "Resolved digest: ${REF}"
echo "==> cosign sign (keyless) --recursive ${REF}"
cosign sign --recursive "${REF}"
echo "==> cosign sign (key) --recursive ${REF}"
cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}"
# Retry wrapper for verification to handle registry propagation delays
retry_verify() {
local cmd="$1"
local attempts=6
local delay=5
local i=1
until eval "$cmd"; do
if [ $i -ge $attempts ]; then
echo "Verification failed after $attempts attempts"
return 1
fi
echo "Verification not yet available. Retry $i/$attempts after ${delay}s..."
sleep $delay
i=$((i+1))
delay=$((delay*2))
# Cap the delay to avoid very long waits
if [ $delay -gt 60 ]; then delay=60; fi
done
return 0
}
echo "==> cosign verify (public key) ${REF}"
if retry_verify "cosign verify --key env://COSIGN_PUBLIC_KEY '${REF}' -o text"; then
VERIFIED_INDEX=true
else
VERIFIED_INDEX=false
fi
echo "==> cosign verify (keyless policy) ${REF}"
if retry_verify "cosign verify --certificate-oidc-issuer '${issuer}' --certificate-identity-regexp '${id_regex}' '${REF}' -o text"; then
VERIFIED_INDEX_KEYLESS=true
else
VERIFIED_INDEX_KEYLESS=false
fi
# Check if verification succeeded
if [ "${VERIFIED_INDEX}" != "true" ] && [ "${VERIFIED_INDEX_KEYLESS}" != "true" ]; then
echo "⚠️ WARNING: Verification not available for ${BASE_IMAGE}:${IMAGE_TAG}"
echo "This may be due to registry propagation delays. Continuing anyway."
fi
) || TAG_FAILED=true
if [ "$TAG_FAILED" = "true" ]; then
echo "⚠️ WARNING: Failed to sign/verify ${BASE_IMAGE}:${IMAGE_TAG}"
FAILED_TAGS+=("${BASE_IMAGE}:${IMAGE_TAG}")
else
echo "✓ Successfully signed and verified ${BASE_IMAGE}:${IMAGE_TAG}"
SUCCESSFUL_TAGS+=("${BASE_IMAGE}:${IMAGE_TAG}")
fi
done
done
# Report summary
echo ""
echo "=========================================="
echo "Sign and Verify Summary"
echo "=========================================="
echo "Successful: ${#SUCCESSFUL_TAGS[@]}"
echo "Failed: ${#FAILED_TAGS[@]}"
echo ""
if [ ${#FAILED_TAGS[@]} -gt 0 ]; then
echo "Failed tags:"
for tag in "${FAILED_TAGS[@]}"; do
echo " - $tag"
done
echo ""
echo "⚠️ WARNING: Some tags failed to sign/verify, but continuing anyway"
else
echo "✓ All images signed and verified successfully!"
fi
shell: bash
post-run:
needs: [pre-run, release-arm, release-amd, create-manifest, sign-and-package]
if: >-
${{
always() &&
needs.pre-run.result == 'success' &&
(needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') &&
(needs.release-amd.result == 'success' || needs.release-amd.result == 'skipped' || needs.release-amd.result == 'failure') &&
(needs.create-manifest.result == 'success' || needs.create-manifest.result == 'skipped' || needs.create-manifest.result == 'failure') &&
(needs.sign-and-package.result == 'success' || needs.sign-and-package.result == 'skipped' || needs.sign-and-package.result == 'failure')
}}
runs-on: ubuntu-latest
permissions: write-all
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v5
with:
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
role-duration-seconds: 3600
aws-region: ${{ secrets.AWS_REGION }}
- name: Verify AWS identity
run: aws sts get-caller-identity
- name: Stop EC2 instances
run: |
aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
echo "EC2 instances stopped"
Reference in New Issue View Git Blame Copy Permalink