From 91a035f4ab4a614490e48e75e6efc88fd95b55a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=A4fer?= Date: Wed, 4 Mar 2026 23:51:10 +0100 Subject: [PATCH 01/12] fix(ci): Use AWS SelfHosted runner to fix pull and install request limit --- .github/workflows/cicd.yml | 806 +++++++++++++++++++------------------ 1 file changed, 425 insertions(+), 381 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index ac498c6..8eb438b 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -1,9 +1,19 @@ -name: CI/CD Pipeline +name: CI/CD Pipeline (AWS Self-Hosted Runners) + +# CI/CD workflow for building, publishing, attesting, signing container images and building release binaries. +# Native multi-arch pipeline using two AWS EC2 self-hosted runners (x86_64 + arm64) to build and push architecture-specific images in parallel, then create multi-arch manifests. +# +# Required secrets: +# - AWS_ACCOUNT_ID, AWS_ROLE_NAME, AWS_REGION +# - EC2_INSTANCE_ID_AMD_RUNNER, EC2_INSTANCE_ID_ARM_RUNNER +# - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN +# - GITHUB_TOKEN +# - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY permissions: contents: write # gh-release packages: write # GHCR push - id-token: write # Keyless-Signatures & Attestations + id-token: write # Keyless-Signatures & Attestations (OIDC) attestations: write # actions/attest-build-provenance security-events: write # upload-sarif actions: read @@ -17,16 +27,16 @@ on: workflow_dispatch: inputs: version: - description: "SemVer version to release (e.g., 1.2.3, no leading 'v')" + description: "Version to release (X.Y.Z or X.Y.Z-rc.N)" required: true type: string publish_latest: - description: "Also publish the 'latest' image tag" + description: "Publish latest tag (non-RC only)" required: true type: boolean default: false publish_minor: - description: "Also publish the 'major.minor' image tag (e.g., 1.2)" + description: "Publish minor tag (X.Y) (non-RC only)" required: true type: boolean default: false @@ -40,10 +50,38 @@ concurrency: cancel-in-progress: true jobs: + # --------------------------------------------------------------------------- + # 1) Start AWS EC2 runner instances + # --------------------------------------------------------------------------- + pre-run: + name: Start AWS EC2 runners + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Start EC2 instances + run: | + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} + aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + echo "EC2 instances started" + + # --------------------------------------------------------------------------- + # 2) Prepare release + # --------------------------------------------------------------------------- prepare: if: github.event_name == 'workflow_dispatch' name: Prepare release (create tag) - runs-on: ubuntu-24.04 + needs: [pre-run] + runs-on: [self-hosted, linux, x64, us-east-1] permissions: contents: write steps: @@ -62,6 +100,7 @@ jobs: echo "Invalid version: $INPUT_VERSION (expected X.Y.Z or X.Y.Z-rc.N)" >&2 exit 1 fi + - name: Create and push tag shell: bash env: @@ -81,14 +120,24 @@ jobs: fi git tag -a "$VERSION" -m "Release $VERSION" git push origin "refs/tags/$VERSION" - release: - if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.actor != 'github-actions[bot]') }} - name: Build and Release - runs-on: ubuntu-24.04 + + # --------------------------------------------------------------------------- + # 3) Build and Release (x86 job) + # --------------------------------------------------------------------------- + build-amd: + name: Build image (linux/amd64) + needs: [pre-run, prepare] + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + runs-on: [self-hosted, linux, x64, us-east-1] timeout-minutes: 120 env: DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + outputs: + tag: ${{ steps.tag.outputs.tag }} + is_rc: ${{ steps.tag.outputs.is_rc }} + major: ${{ steps.tag.outputs.major }} + minor: ${{ steps.tag.outputs.minor }} steps: - name: Checkout code @@ -96,15 +145,65 @@ jobs: with: fetch-depth: 0 + - name: Monitor storage space + shell: bash + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo "Disk usage >= ${THRESHOLD}%, pruning docker..." + echo y | docker system prune -a || true + else + echo "Disk usage < ${THRESHOLD}%, no action needed." + fi + + - name: Determine tag + rc/major/minor + id: tag + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + TAG="$INPUT_VERSION" + else + TAG="${{ github.ref_name }}" + fi + + if ! [[ "$TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then + echo "Invalid tag: $TAG" >&2 + exit 1 + fi + + IS_RC="false" + if [[ "$TAG" =~ -rc\.[0-9]+$ ]]; then + IS_RC="true" + fi + + MAJOR="$(echo "$TAG" | cut -d. -f1)" + MINOR="$(echo "$TAG" | cut -d. -f1,2)" + + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "is_rc=$IS_RC" >> "$GITHUB_OUTPUT" + echo "major=$MAJOR" >> "$GITHUB_OUTPUT" + echo "minor=$MINOR" >> "$GITHUB_OUTPUT" + + echo "TAG=$TAG" >> $GITHUB_ENV + echo "IS_RC=$IS_RC" >> $GITHUB_ENV + echo "MAJOR_TAG=$MAJOR" >> $GITHUB_ENV + echo "MINOR_TAG=$MINOR" >> $GITHUB_ENV + - name: Capture created timestamp run: echo "IMAGE_CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV shell: bash - - name: Set up QEMU - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + #- name: Set up QEMU + # uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + #- name: Set up Docker Buildx + # uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to Docker Hub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 @@ -121,78 +220,231 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Normalize image names to lowercase + shell: bash run: | set -euo pipefail echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" - shell: bash - - name: Extract tag name + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + # Build ONLY amd64 and push arch-specific tag suffixes used later for manifest creation. + - name: Build and push (amd64 -> *:amd64-TAG) + id: build_amd + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: true + platforms: linux/amd64 + tags: | + ${{ env.GHCR_IMAGE }}:amd64-${{ env.TAG }} + ${{ env.DOCKERHUB_IMAGE }}:amd64-${{ env.TAG }} + cache-from: type=gha,scope=${{ github.repository }}-amd64 + cache-to: type=gha,mode=max,scope=${{ github.repository }}-amd64 + + # --------------------------------------------------------------------------- + # 4) Build ARM64 image natively on ARM runner + # --------------------------------------------------------------------------- + build-arm: + name: Build image (linux/arm64) + needs: [pre-run, prepare] + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + runs-on: [self-hosted, linux, arm64, us-east-1] # NOTE: ensure label exists on runner + timeout-minutes: 120 + env: + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + + - name: Monitor storage space + shell: bash + run: | + THRESHOLD=75 + USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g') + echo "Used space: $USED_SPACE%" + if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then + echo y | docker system prune -a || true + fi + + - name: Determine tag + validate format + shell: bash env: EVENT_NAME: ${{ github.event_name }} INPUT_VERSION: ${{ inputs.version }} run: | + set -euo pipefail if [ "$EVENT_NAME" = "workflow_dispatch" ]; then - echo "TAG=${INPUT_VERSION}" >> $GITHUB_ENV + TAG="$INPUT_VERSION" else - echo "TAG=${{ github.ref_name }}" >> $GITHUB_ENV + TAG="${{ github.ref_name }}" fi - shell: bash - - name: Validate pushed tag format (no leading 'v') - if: ${{ github.event_name == 'push' }} - shell: bash - env: - TAG_GOT: ${{ env.TAG }} - run: | - set -euo pipefail - if [[ "$TAG_GOT" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then - echo "Tag OK: $TAG_GOT" - exit 0 + if ! [[ "$TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then + echo "Invalid tag: $TAG" >&2 + exit 1 fi - echo "ERROR: Tag '$TAG_GOT' is not allowed. Use 'X.Y.Z' or 'X.Y.Z-rc.N' (no leading 'v')." >&2 - exit 1 - - name: Wait for tag to be visible (dispatch only) - if: ${{ github.event_name == 'workflow_dispatch' }} + + echo "TAG=$TAG" >> $GITHUB_ENV + + - name: Log in to Docker Hub + if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Normalize image names to lowercase + shell: bash run: | set -euo pipefail - for i in {1..90}; do - if git ls-remote --tags origin "refs/tags/${TAG}" | grep -qE "refs/tags/${TAG}$"; then - echo "Tag ${TAG} is visible on origin"; exit 0 + echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" + echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + # Build ONLY arm64 and push arch-specific tag suffixes used later for manifest creation. + - name: Build and push (arm64 -> *:arm64-TAG) + id: build_arm + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: true + platforms: linux/arm64 + tags: | + ${{ env.GHCR_IMAGE }}:arm64-${{ env.TAG }} + ${{ env.DOCKERHUB_IMAGE }}:arm64-${{ env.TAG }} + cache-from: type=gha,scope=${{ github.repository }}-arm64 + cache-to: type=gha,mode=max,scope=${{ github.repository }}-arm64 + + # --------------------------------------------------------------------------- + # 5) Create and push multi-arch manifests (TAG, plus optional latest/major/minor) + # --------------------------------------------------------------------------- + create-manifest: + name: Create multi-arch manifests + needs: [build-amd, build-arm] + if: ${{ needs.build-amd.result == 'success' && needs.build-arm.result == 'success' }} + runs-on: [self-hosted, linux, x64, us-east-1] # NOTE: ensure label exists on runner + timeout-minutes: 30 + env: + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + TAG: ${{ needs.build-amd.outputs.tag }} + IS_RC: ${{ needs.build-amd.outputs.is_rc }} + MAJOR_TAG: ${{ needs.build-amd.outputs.major }} + MINOR_TAG: ${{ needs.build-amd.outputs.minor }} + # workflow_dispatch controls are respected only here (tagging policy) + PUBLISH_LATEST: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_latest || vars.PUBLISH_LATEST }} + PUBLISH_MINOR: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_minor || vars.PUBLISH_MINOR }} + steps: + - name: Log in to Docker Hub + if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Normalize image names to lowercase + shell: bash + run: | + set -euo pipefail + echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" + echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" + + - name: Create manifest for GHCR (:TAG) + shell: bash + run: | + set -euo pipefail + docker manifest create "${GHCR_IMAGE}:${TAG}" \ + --amend "${GHCR_IMAGE}:amd64-${TAG}" \ + --amend "${GHCR_IMAGE}:arm64-${TAG}" + docker manifest push "${GHCR_IMAGE}:${TAG}" + + - name: Create manifest for Docker Hub (:TAG) + if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} + shell: bash + run: | + set -euo pipefail + docker manifest create "${DOCKERHUB_IMAGE}:${TAG}" \ + --amend "${DOCKERHUB_IMAGE}:amd64-${TAG}" \ + --amend "${DOCKERHUB_IMAGE}:arm64-${TAG}" + docker manifest push "${DOCKERHUB_IMAGE}:${TAG}" + + # Optional tags for non-RC releases: latest, major, minor + - name: Publish additional tags (non-RC only) + if: ${{ env.IS_RC != 'true' }} + shell: bash + run: | + set -euo pipefail + + tags_to_publish=() + tags_to_publish+=("${MAJOR_TAG}") + if [ "${PUBLISH_MINOR}" = "true" ]; then + tags_to_publish+=("${MINOR_TAG}") + fi + if [ "${PUBLISH_LATEST}" = "true" ]; then + tags_to_publish+=("latest") + fi + + for t in "${tags_to_publish[@]}"; do + echo "Publishing GHCR tag ${t} -> ${TAG}" + docker manifest create "${GHCR_IMAGE}:${t}" \ + --amend "${GHCR_IMAGE}:amd64-${TAG}" \ + --amend "${GHCR_IMAGE}:arm64-${TAG}" + docker manifest push "${GHCR_IMAGE}:${t}" + + if [ -n "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" ]; then + echo "Publishing Docker Hub tag ${t} -> ${TAG}" + docker manifest create "${DOCKERHUB_IMAGE}:${t}" \ + --amend "${DOCKERHUB_IMAGE}:amd64-${TAG}" \ + --amend "${DOCKERHUB_IMAGE}:arm64-${TAG}" + docker manifest push "${DOCKERHUB_IMAGE}:${t}" fi - echo "Tag not yet visible, retrying... ($i/90)" - sleep 2 done - echo "Tag ${TAG} not visible after waiting"; exit 1 - shell: bash - - name: Update version in main.go - run: | - TAG=${{ env.TAG }} - if [ -f main.go ]; then - sed -i 's/version_replaceme/'"$TAG"'/' main.go - echo "Updated main.go with version $TAG" - else - echo "main.go not found" - fi + # --------------------------------------------------------------------------- + # 6) Sign/attest + build binaries + draft release (x86 runner) + # --------------------------------------------------------------------------- + sign-and-release: + name: Sign, attest, and release + needs: [create-manifest, build-amd] + if: ${{ needs.create-manifest.result == 'success' && needs.build-amd.result == 'success' }} + runs-on: [self-hosted, linux, x64, us-east-1] # NOTE: ensure label exists on runner + timeout-minutes: 120 + env: + DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + TAG: ${{ needs.build-amd.outputs.tag }} + IS_RC: ${{ needs.build-amd.outputs.is_rc }} + steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 - - name: Ensure repository is at the tagged commit (dispatch only) - if: ${{ github.event_name == 'workflow_dispatch' }} - run: | - set -euo pipefail - git fetch --tags --force - git checkout "refs/tags/${TAG}" - echo "Checked out $(git rev-parse --short HEAD) for tag ${TAG}" - shell: bash - - - name: Detect release candidate (rc) - run: | - set -euo pipefail - if [[ "${TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then - echo "IS_RC=true" >> $GITHUB_ENV - else - echo "IS_RC=false" >> $GITHUB_ENV - fi + - name: Capture created timestamp + run: echo "IMAGE_CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV shell: bash - name: Install Go @@ -200,234 +452,113 @@ jobs: with: go-version-file: go.mod - - name: Resolve publish-latest flag - env: - EVENT_NAME: ${{ github.event_name }} - PL_INPUT: ${{ inputs.publish_latest }} - PL_VAR: ${{ vars.PUBLISH_LATEST }} - run: | - set -euo pipefail - val="false" - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then - if [ "${PL_INPUT}" = "true" ]; then val="true"; fi - else - if [ "${PL_VAR}" = "true" ]; then val="true"; fi - fi - echo "PUBLISH_LATEST=$val" >> $GITHUB_ENV - shell: bash - - - name: Resolve publish-minor flag - env: - EVENT_NAME: ${{ github.event_name }} - PM_INPUT: ${{ inputs.publish_minor }} - PM_VAR: ${{ vars.PUBLISH_MINOR }} - run: | - set -euo pipefail - val="false" - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then - if [ "${PM_INPUT}" = "true" ]; then val="true"; fi - else - if [ "${PM_VAR}" = "true" ]; then val="true"; fi - fi - echo "PUBLISH_MINOR=$val" >> $GITHUB_ENV - shell: bash - - - name: Cache Go modules - if: ${{ hashFiles('**/go.sum') != '' }} - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - name: Log in to Docker Hub + if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Go vet & test - if: ${{ hashFiles('**/go.mod') != '' }} - run: | - go version - go vet ./... - go test ./... -race -covermode=atomic - shell: bash + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - name: Resolve license fallback - run: echo "IMAGE_LICENSE=${{ github.event.repository.license.spdx_id || 'NOASSERTION' }}" >> $GITHUB_ENV - shell: bash + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Resolve registries list (GHCR always, Docker Hub only if creds) + - name: Normalize image names to lowercase shell: bash run: | set -euo pipefail - images="${GHCR_IMAGE}" - if [ -n "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" ] && [ -n "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - images="${images}\n${DOCKERHUB_IMAGE}" + echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" + echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" + + - name: Resolve multi-arch digest refs (by TAG) + shell: bash + run: | + set -euo pipefail + GHCR_DIGEST="$(docker buildx imagetools inspect "${GHCR_IMAGE}:${TAG}" --format '{{.Manifest.Digest}}')" + echo "GHCR_REF=${GHCR_IMAGE}@${GHCR_DIGEST}" >> $GITHUB_ENV + echo "Resolved GHCR_REF=${GHCR_IMAGE}@${GHCR_DIGEST}" + if [ -n "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" ]; then + DH_DIGEST="$(docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:${TAG}" --format '{{.Manifest.Digest}}')" + echo "DH_REF=${DOCKERHUB_IMAGE}@${DH_DIGEST}" >> $GITHUB_ENV + echo "Resolved DH_REF=${DOCKERHUB_IMAGE}@${DH_DIGEST}" fi - { - echo 'IMAGE_LIST<> "$GITHUB_ENV" - - name: Docker meta - id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 - with: - images: ${{ env.IMAGE_LIST }} - tags: | - type=semver,pattern={{version}},value=${{ env.TAG }} - type=semver,pattern={{major}}.{{minor}},value=${{ env.TAG }},enable=${{ env.PUBLISH_MINOR == 'true' && env.IS_RC != 'true' }} - type=raw,value=latest,enable=${{ env.IS_RC != 'true' }} - flavor: | - latest=false - labels: | - org.opencontainers.image.title=${{ github.event.repository.name }} - org.opencontainers.image.version=${{ env.TAG }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.url=${{ github.event.repository.html_url }} - org.opencontainers.image.documentation=${{ github.event.repository.html_url }} - org.opencontainers.image.description=${{ github.event.repository.description }} - org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }} - org.opencontainers.image.created=${{ env.IMAGE_CREATED }} - org.opencontainers.image.ref.name=${{ env.TAG }} - org.opencontainers.image.authors=${{ github.repository_owner }} - - name: Echo build config (non-secret) + + - name: Extract digests for attestation shell: bash - env: - IMAGE_TITLE: ${{ github.event.repository.name }} - IMAGE_VERSION: ${{ env.TAG }} - IMAGE_REVISION: ${{ github.sha }} - IMAGE_SOURCE_URL: ${{ github.event.repository.html_url }} - IMAGE_URL: ${{ github.event.repository.html_url }} - IMAGE_DESCRIPTION: ${{ github.event.repository.description }} - IMAGE_LICENSE: ${{ env.IMAGE_LICENSE }} - DOCKERHUB_IMAGE: ${{ env.DOCKERHUB_IMAGE }} - GHCR_IMAGE: ${{ env.GHCR_IMAGE }} - DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USERNAME }} - REPO: ${{ github.repository }} - OWNER: ${{ github.repository_owner }} - WORKFLOW_REF: ${{ github.workflow_ref }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail - echo "=== OCI Label Values ===" - echo "org.opencontainers.image.title=${IMAGE_TITLE}" - echo "org.opencontainers.image.version=${IMAGE_VERSION}" - echo "org.opencontainers.image.revision=${IMAGE_REVISION}" - echo "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" - echo "org.opencontainers.image.url=${IMAGE_URL}" - echo "org.opencontainers.image.description=${IMAGE_DESCRIPTION}" - echo "org.opencontainers.image.licenses=${IMAGE_LICENSE}" - echo - echo "=== Images ===" - echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE}" - echo "GHCR_IMAGE=${GHCR_IMAGE}" - echo "DOCKER_HUB_USERNAME=${DOCKER_HUB_USER}" - echo - echo "=== GitHub Kontext ===" - echo "repository=${REPO}" - echo "owner=${OWNER}" - echo "workflow_ref=${WORKFLOW_REF}" - echo "ref=${REF}" - echo "ref_name=${REF_NAME}" - echo "run_url=${RUN_URL}" - echo - echo "=== docker/metadata-action outputs (Tags/Labels), raw ===" - echo "::group::tags" - echo "${{ steps.meta.outputs.tags }}" - echo "::endgroup::" - echo "::group::labels" - echo "${{ steps.meta.outputs.labels }}" - echo "::endgroup::" - - name: Build and push (Docker Hub + GHCR) - id: build - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha,scope=${{ github.repository }} - cache-to: type=gha,mode=max,scope=${{ github.repository }} - provenance: mode=max - sbom: true + echo "GHCR_DIGEST=${GHCR_REF#*@}" >> $GITHUB_ENV + if [ -n "${DH_REF:-}" ]; then + echo "DH_DIGEST=${DH_REF#*@}" >> $GITHUB_ENV + fi - - name: Compute image digest refs - run: | - echo "DIGEST=${{ steps.build.outputs.digest }}" >> $GITHUB_ENV - echo "GHCR_REF=$GHCR_IMAGE@${{ steps.build.outputs.digest }}" >> $GITHUB_ENV - echo "DH_REF=$DOCKERHUB_IMAGE@${{ steps.build.outputs.digest }}" >> $GITHUB_ENV - echo "Built digest: ${{ steps.build.outputs.digest }}" - shell: bash - - - name: Attest build provenance (GHCR) - id: attest-ghcr + - name: Attest build provenance (GHCR) (digest) uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 with: subject-name: ${{ env.GHCR_IMAGE }} - subject-digest: ${{ steps.build.outputs.digest }} + subject-digest: ${{ env.GHCR_DIGEST }} push-to-registry: true show-summary: true - name: Attest build provenance (Docker Hub) continue-on-error: true - id: attest-dh + if: ${{ env.DH_DIGEST != '' }} uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 with: subject-name: index.docker.io/fosrl/${{ github.event.repository.name }} - subject-digest: ${{ steps.build.outputs.digest }} + subject-digest: ${{ env.DH_DIGEST }} push-to-registry: true show-summary: true - name: Install cosign uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 with: - cosign-release: 'v3.0.2' + cosign-release: "v3.0.2" - name: Sanity check cosign private key env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + shell: bash run: | set -euo pipefail cosign public-key --key env://COSIGN_PRIVATE_KEY >/dev/null - shell: bash - - name: Sign GHCR image (digest) with key (recursive) - env: - COSIGN_YES: "true" - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} - run: | - set -euo pipefail - echo "Signing ${GHCR_REF} (digest) recursively with provided key" - cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${GHCR_REF}" - echo "Waiting 30 seconds for signatures to propagate..." - sleep 30 - shell: bash - - - name: Generate SBOM (SPDX JSON) + - name: Generate SBOM (SPDX JSON) from GHCR digest uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 with: - image-ref: ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} + image-ref: ${{ env.GHCR_REF }} format: spdx-json output: sbom.spdx.json - - name: Validate SBOM JSON - run: jq -e . sbom.spdx.json >/dev/null + - name: Validate + minify SBOM JSON shell: bash + run: | + set -euo pipefail + jq -e . sbom.spdx.json >/dev/null + jq -c . sbom.spdx.json > sbom.min.json && mv sbom.min.json sbom.spdx.json - - name: Minify SBOM JSON (optional hardening) - run: jq -c . sbom.spdx.json > sbom.min.json && mv sbom.min.json sbom.spdx.json - shell: bash - - - name: Create SBOM attestation (GHCR, private key) + - name: Sign GHCR digest (key, recursive) env: COSIGN_YES: "true" COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + shell: bash + run: | + set -euo pipefail + cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${GHCR_REF}" + sleep 20 + + - name: Create SBOM attestation (GHCR, key) + env: + COSIGN_YES: "true" + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + shell: bash run: | set -euo pipefail cosign attest \ @@ -435,171 +566,46 @@ jobs: --type spdxjson \ --predicate sbom.spdx.json \ "${GHCR_REF}" - shell: bash - - - name: Create SBOM attestation (Docker Hub, private key) - continue-on-error: true - env: - COSIGN_YES: "true" - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} - COSIGN_DOCKER_MEDIA_TYPES: "1" - run: | - set -euo pipefail - cosign attest \ - --key env://COSIGN_PRIVATE_KEY \ - --type spdxjson \ - --predicate sbom.spdx.json \ - "${DH_REF}" - shell: bash - name: Keyless sign & verify GHCR digest (OIDC) env: COSIGN_YES: "true" - WORKFLOW_REF: ${{ github.workflow_ref }} # owner/repo/.github/workflows/@refs/tags/ + WORKFLOW_REF: ${{ github.workflow_ref }} ISSUER: https://token.actions.githubusercontent.com + shell: bash run: | set -euo pipefail - echo "Keyless signing ${GHCR_REF}" cosign sign --rekor-url https://rekor.sigstore.dev --recursive "${GHCR_REF}" - echo "Verify keyless (OIDC) signature policy on ${GHCR_REF}" cosign verify \ --certificate-oidc-issuer "${ISSUER}" \ --certificate-identity "https://github.com/${WORKFLOW_REF}" \ "${GHCR_REF}" -o text - shell: bash - - name: Sign Docker Hub image (digest) with key (recursive) - continue-on-error: true - env: - COSIGN_YES: "true" - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} - COSIGN_DOCKER_MEDIA_TYPES: "1" - run: | - set -euo pipefail - echo "Signing ${DH_REF} (digest) recursively with provided key (Docker media types fallback)" - cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${DH_REF}" - shell: bash - - - name: Keyless sign & verify Docker Hub digest (OIDC) - continue-on-error: true - env: - COSIGN_YES: "true" - ISSUER: https://token.actions.githubusercontent.com - COSIGN_DOCKER_MEDIA_TYPES: "1" - run: | - set -euo pipefail - echo "Keyless signing ${DH_REF} (force public-good Rekor)" - cosign sign --rekor-url https://rekor.sigstore.dev --recursive "${DH_REF}" - echo "Keyless verify via Rekor (strict identity)" - if ! cosign verify \ - --rekor-url https://rekor.sigstore.dev \ - --certificate-oidc-issuer "${ISSUER}" \ - --certificate-identity "https://github.com/${{ github.workflow_ref }}" \ - "${DH_REF}" -o text; then - echo "Rekor verify failed — retry offline bundle verify (no Rekor)" - if ! cosign verify \ - --offline \ - --certificate-oidc-issuer "${ISSUER}" \ - --certificate-identity "https://github.com/${{ github.workflow_ref }}" \ - "${DH_REF}" -o text; then - echo "Offline bundle verify failed — ignore tlog (TEMP for debugging)" - cosign verify \ - --insecure-ignore-tlog=true \ - --certificate-oidc-issuer "${ISSUER}" \ - --certificate-identity "https://github.com/${{ github.workflow_ref }}" \ - "${DH_REF}" -o text || true - fi - fi - name: Verify signature (public key) GHCR digest + tag env: COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} + shell: bash run: | set -euo pipefail - TAG_VAR="${TAG}" - echo "Verifying (digest) ${GHCR_REF}" - cosign verify --key env://COSIGN_PUBLIC_KEY "$GHCR_REF" -o text - echo "Verifying (tag) $GHCR_IMAGE:$TAG_VAR" - cosign verify --key env://COSIGN_PUBLIC_KEY "$GHCR_IMAGE:$TAG_VAR" -o text - shell: bash + cosign verify --key env://COSIGN_PUBLIC_KEY "${GHCR_REF}" -o text + cosign verify --key env://COSIGN_PUBLIC_KEY "${GHCR_IMAGE}:${TAG}" -o text - name: Verify SBOM attestation (GHCR) env: COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} - run: cosign verify-attestation --key env://COSIGN_PUBLIC_KEY --type spdxjson "$GHCR_REF" -o text + run: cosign verify-attestation --key env://COSIGN_PUBLIC_KEY --type spdxjson "${GHCR_REF}" -o text shell: bash - - name: Verify SLSA provenance (GHCR) - env: - ISSUER: https://token.actions.githubusercontent.com - WFREF: ${{ github.workflow_ref }} - run: | - set -euo pipefail - # (optional) show which predicate types are present to aid debugging - cosign download attestation "$GHCR_REF" \ - | jq -r '.payload | @base64d | fromjson | .predicateType' | sort -u || true - # Verify the SLSA v1 provenance attestation (predicate URL) - cosign verify-attestation \ - --type 'https://slsa.dev/provenance/v1' \ - --certificate-oidc-issuer "$ISSUER" \ - --certificate-identity "https://github.com/${WFREF}" \ - --rekor-url https://rekor.sigstore.dev \ - "$GHCR_REF" -o text - shell: bash - - - name: Verify signature (public key) Docker Hub digest - continue-on-error: true - env: - COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} - COSIGN_DOCKER_MEDIA_TYPES: "1" - run: | - set -euo pipefail - echo "Verifying (digest) ${DH_REF} with Docker media types" - cosign verify --key env://COSIGN_PUBLIC_KEY "${DH_REF}" -o text - shell: bash - - - name: Verify signature (public key) Docker Hub tag - continue-on-error: true - env: - COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} - COSIGN_DOCKER_MEDIA_TYPES: "1" - run: | - set -euo pipefail - echo "Verifying (tag) $DOCKERHUB_IMAGE:$TAG with Docker media types" - cosign verify --key env://COSIGN_PUBLIC_KEY "$DOCKERHUB_IMAGE:$TAG" -o text - shell: bash - - # - name: Trivy scan (GHCR image) - # id: trivy - # uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 - # with: - # image-ref: ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} - # format: sarif - # output: trivy-ghcr.sarif - # ignore-unfixed: true - # vuln-type: os,library - # severity: CRITICAL,HIGH - # exit-code: ${{ (vars.TRIVY_FAIL || '0') }} - - # - name: Upload SARIF,trivy - # if: ${{ always() && hashFiles('trivy-ghcr.sarif') != '' }} - # uses: github/codeql-action/upload-sarif@fdbfb4d2750291e159f0156def62b853c2798ca2 # v4.31.5 - # with: - # sarif_file: trivy-ghcr.sarif - # category: Image Vulnerability Scan - - name: Build binaries env: CGO_ENABLED: "0" GOFLAGS: "-trimpath" + shell: bash run: | set -euo pipefail - TAG_VAR="${TAG}" - make -j 10 go-build-release tag=$TAG_VAR - shell: bash + make -j 10 go-build-release tag="${TAG}" - - name: Create GitHub Release + - name: Create GitHub Release (draft) uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe # v2.4.2 with: tag_name: ${{ env.TAG }} @@ -613,4 +619,42 @@ jobs: ## Container Images - GHCR: `${{ env.GHCR_REF }}` - Docker Hub: `${{ env.DH_REF || 'N/A' }}` - **Digest:** `${{ steps.build.outputs.digest }}` + **Tag:** `${{ env.TAG }}` + + # --------------------------------------------------------------------------- + # 7) Stop AWS EC2 runner instances + # --------------------------------------------------------------------------- + post-run: + name: Stop AWS EC2 runners + needs: [pre-run, prepare, build-amd, build-arm, create-manifest, sign-and-release] + if: >- + ${{ + always() && + (needs.pre-run.result == 'success' && + (needs.build-amd.result == 'success' || needs.build-amd.result == 'failure' || needs.build-amd.result == 'cancelled') || + needs.build-amd.result != 'skipped') && + (needs.build-arm.result == 'success' || needs.build-arm.result == 'failure' || needs.build-arm.result == 'cancelled') || + needs.build-arm.result != 'skipped') && + (needs.create-manifest.result == 'success' || needs.create-manifest.result == 'failure' || needs.create-manifest.result == 'cancelled') || + needs.create-manifest.result != 'skipped') && + (needs.sign-and-release.result == 'success' || needs.sign-and-release.result == 'failure' || needs.sign-and-release.result == 'cancelled') || + needs.sign-and-release.result != 'skipped') + }} + runs-on: ubuntu-latest + permissions: write-all + steps: + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-duration-seconds: 3600 + aws-region: ${{ secrets.AWS_REGION }} + + - name: Verify AWS identity + run: aws sts get-caller-identity + + - name: Stop EC2 instances + run: | + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }} + aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }} + echo "EC2 instances stopped" From 068145c539e128c0e97f725a31e7c1971936eb2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=A4fer?= Date: Fri, 6 Mar 2026 19:12:36 +0100 Subject: [PATCH 02/12] fix(ci): Refactor CI/CD workflow for AWS and image management Updated CI/CD workflow to improve AWS role handling and image tagging. --- .github/workflows/cicd.yml | 461 +++++++++++++++++++++++++++++-------- 1 file changed, 367 insertions(+), 94 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 8eb438b..815c400 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -57,11 +57,20 @@ jobs: name: Start AWS EC2 runners runs-on: ubuntu-latest permissions: write-all + outputs: + image_created: ${{ steps.created.outputs.image_created }} steps: + - name: Capture created timestamp (shared) + id: created + shell: bash + run: | + set -euo pipefail + echo "image_created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" + - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} @@ -81,7 +90,7 @@ jobs: if: github.event_name == 'workflow_dispatch' name: Prepare release (create tag) needs: [pre-run] - runs-on: [self-hosted, linux, x64, us-east-1] + runs-on: [self-hosted, linux, x64] permissions: contents: write steps: @@ -128,11 +137,14 @@ jobs: name: Build image (linux/amd64) needs: [pre-run, prepare] if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} - runs-on: [self-hosted, linux, x64, us-east-1] + runs-on: [self-hosted, linux, x64] timeout-minutes: 120 env: - DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + DOCKERHUB_IMAGE: docker.io/${{ github.repository_owner }}/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + IMAGE_LICENSE: ${{ github.event.repository.license.spdx_id || 'NOASSERTION' }} + IMAGE_CREATED: ${{ needs.pre-run.outputs.image_created }} + outputs: tag: ${{ steps.tag.outputs.tag }} is_rc: ${{ steps.tag.outputs.is_rc }} @@ -195,9 +207,29 @@ jobs: echo "MAJOR_TAG=$MAJOR" >> $GITHUB_ENV echo "MINOR_TAG=$MINOR" >> $GITHUB_ENV - - name: Capture created timestamp - run: echo "IMAGE_CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV + - name: Wait for tag to be visible (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} shell: bash + run: | + set -euo pipefail + for i in {1..90}; do + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -qE "refs/tags/${TAG}$"; then + echo "Tag ${TAG} is visible on origin"; exit 0 + fi + echo "Tag not yet visible, retrying... ($i/90)" + sleep 2 + done + echo "Tag ${TAG} not visible after waiting" >&2 + exit 1 + + - name: Ensure repository is at the tagged commit (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + set -euo pipefail + git fetch --tags --force + git checkout "refs/tags/${TAG}" + echo "Checked out $(git rev-parse --short HEAD) for tag ${TAG}" #- name: Set up QEMU # uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 @@ -240,6 +272,18 @@ jobs: tags: | ${{ env.GHCR_IMAGE }}:amd64-${{ env.TAG }} ${{ env.DOCKERHUB_IMAGE }}:amd64-${{ env.TAG }} + labels: | + org.opencontainers.image.title=${{ github.event.repository.name }} + org.opencontainers.image.version=${{ env.TAG }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.url=${{ github.event.repository.html_url }} + org.opencontainers.image.documentation=${{ github.event.repository.html_url }} + org.opencontainers.image.description=${{ github.event.repository.description }} + org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }} + org.opencontainers.image.created=${{ env.IMAGE_CREATED }} + org.opencontainers.image.ref.name=${{ env.TAG }} + org.opencontainers.image.authors=${{ github.repository_owner }} cache-from: type=gha,scope=${{ github.repository }}-amd64 cache-to: type=gha,mode=max,scope=${{ github.repository }}-amd64 @@ -250,11 +294,13 @@ jobs: name: Build image (linux/arm64) needs: [pre-run, prepare] if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} - runs-on: [self-hosted, linux, arm64, us-east-1] # NOTE: ensure label exists on runner + runs-on: [self-hosted, linux, arm64] # NOTE: ensure label exists on runner timeout-minutes: 120 env: - DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + DOCKERHUB_IMAGE: docker.io/${{ github.repository_owner }}/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + IMAGE_LICENSE: ${{ github.event.repository.license.spdx_id || 'NOASSERTION' }} + IMAGE_CREATED: ${{ needs.pre-run.outputs.image_created }} steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -291,8 +337,31 @@ jobs: echo "TAG=$TAG" >> $GITHUB_ENV + - name: Wait for tag to be visible (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + set -euo pipefail + for i in {1..90}; do + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -qE "refs/tags/${TAG}$"; then + echo "Tag ${TAG} is visible on origin"; exit 0 + fi + echo "Tag not yet visible, retrying... ($i/90)" + sleep 2 + done + echo "Tag ${TAG} not visible after waiting" >&2 + exit 1 + + - name: Ensure repository is at the tagged commit (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + set -euo pipefail + git fetch --tags --force + git checkout "refs/tags/${TAG}" + echo "Checked out $(git rev-parse --short HEAD) for tag ${TAG}" + - name: Log in to Docker Hub - if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: docker.io @@ -327,31 +396,86 @@ jobs: tags: | ${{ env.GHCR_IMAGE }}:arm64-${{ env.TAG }} ${{ env.DOCKERHUB_IMAGE }}:arm64-${{ env.TAG }} + labels: | + org.opencontainers.image.title=${{ github.event.repository.name }} + org.opencontainers.image.version=${{ env.TAG }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.url=${{ github.event.repository.html_url }} + org.opencontainers.image.documentation=${{ github.event.repository.html_url }} + org.opencontainers.image.description=${{ github.event.repository.description }} + org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }} + org.opencontainers.image.created=${{ env.IMAGE_CREATED }} + org.opencontainers.image.ref.name=${{ env.TAG }} + org.opencontainers.image.authors=${{ github.repository_owner }} cache-from: type=gha,scope=${{ github.repository }}-arm64 cache-to: type=gha,mode=max,scope=${{ github.repository }}-arm64 # --------------------------------------------------------------------------- - # 5) Create and push multi-arch manifests (TAG, plus optional latest/major/minor) + # 4b) Build ARMv7 image (linux/arm/v7) on arm runner via QEMU # --------------------------------------------------------------------------- - create-manifest: - name: Create multi-arch manifests - needs: [build-amd, build-arm] - if: ${{ needs.build-amd.result == 'success' && needs.build-arm.result == 'success' }} - runs-on: [self-hosted, linux, x64, us-east-1] # NOTE: ensure label exists on runner - timeout-minutes: 30 + build-armv7: + name: Build image (linux/arm/v7) + needs: [pre-run, prepare] + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + runs-on: [self-hosted, linux, arm64] + timeout-minutes: 120 env: - DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + DOCKERHUB_IMAGE: docker.io/${{ github.repository_owner }}/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} - TAG: ${{ needs.build-amd.outputs.tag }} - IS_RC: ${{ needs.build-amd.outputs.is_rc }} - MAJOR_TAG: ${{ needs.build-amd.outputs.major }} - MINOR_TAG: ${{ needs.build-amd.outputs.minor }} - # workflow_dispatch controls are respected only here (tagging policy) - PUBLISH_LATEST: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_latest || vars.PUBLISH_LATEST }} - PUBLISH_MINOR: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_minor || vars.PUBLISH_MINOR }} + IMAGE_LICENSE: ${{ github.event.repository.license.spdx_id || 'NOASSERTION' }} + IMAGE_CREATED: ${{ needs.pre-run.outputs.image_created }} steps: + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + + - name: Determine tag + validate format + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + TAG="$INPUT_VERSION" + else + TAG="${{ github.ref_name }}" + fi + + if ! [[ "$TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then + echo "Invalid tag: $TAG" >&2 + exit 1 + fi + + echo "TAG=$TAG" >> $GITHUB_ENV + + - name: Wait for tag to be visible (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + set -euo pipefail + for i in {1..90}; do + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -qE "refs/tags/${TAG}$"; then + echo "Tag ${TAG} is visible on origin"; exit 0 + fi + echo "Tag not yet visible, retrying... ($i/90)" + sleep 2 + done + echo "Tag ${TAG} not visible after waiting" >&2 + exit 1 + + - name: Ensure repository is at the tagged commit (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + set -euo pipefail + git fetch --tags --force + git checkout "refs/tags/${TAG}" + echo "Checked out $(git rev-parse --short HEAD) for tag ${TAG}" + - name: Log in to Docker Hub - if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: docker.io @@ -372,55 +496,124 @@ jobs: echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" - - name: Create manifest for GHCR (:TAG) + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Build and push (arm/v7 -> *:armv7-TAG) + id: build_armv7 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: true + platforms: linux/arm/v7 + tags: | + ${{ env.GHCR_IMAGE }}:armv7-${{ env.TAG }} + ${{ env.DOCKERHUB_IMAGE }}:armv7-${{ env.TAG }} + labels: | + org.opencontainers.image.title=${{ github.event.repository.name }} + org.opencontainers.image.version=${{ env.TAG }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.url=${{ github.event.repository.html_url }} + org.opencontainers.image.documentation=${{ github.event.repository.html_url }} + org.opencontainers.image.description=${{ github.event.repository.description }} + org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }} + org.opencontainers.image.created=${{ env.IMAGE_CREATED }} + org.opencontainers.image.ref.name=${{ env.TAG }} + org.opencontainers.image.authors=${{ github.repository_owner }} + cache-from: type=gha,scope=${{ github.repository }}-armv7 + cache-to: type=gha,mode=max,scope=${{ github.repository }}-armv7 + + # --------------------------------------------------------------------------- + # 5) Create and push multi-arch manifests (TAG, plus optional latest/major/minor) + # --------------------------------------------------------------------------- + create-manifest: + name: Create multi-arch manifests + needs: [build-amd, build-arm, build-armv7] + if: ${{ needs.build-amd.result == 'success' && needs.build-arm.result == 'success' && needs.build-armv7.result == 'success' }} + runs-on: [self-hosted, linux, x64] # NOTE: ensure label exists on runner + timeout-minutes: 30 + env: + DOCKERHUB_IMAGE: docker.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + TAG: ${{ needs.build-amd.outputs.tag }} + IS_RC: ${{ needs.build-amd.outputs.is_rc }} + MAJOR_TAG: ${{ needs.build-amd.outputs.major }} + MINOR_TAG: ${{ needs.build-amd.outputs.minor }} + # workflow_dispatch controls are respected only here (tagging policy) + #PUBLISH_LATEST: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_latest || vars.PUBLISH_LATEST }} + #PUBLISH_MINOR: ${{ github.event_name == 'workflow_dispatch' && inputs.publish_minor || vars.PUBLISH_MINOR }} + steps: + - name: Log in to Docker Hub + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: docker.io + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Normalize image names to lowercase shell: bash run: | set -euo pipefail - docker manifest create "${GHCR_IMAGE}:${TAG}" \ - --amend "${GHCR_IMAGE}:amd64-${TAG}" \ - --amend "${GHCR_IMAGE}:arm64-${TAG}" - docker manifest push "${GHCR_IMAGE}:${TAG}" + echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" + echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" - - name: Create manifest for Docker Hub (:TAG) - if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} + - name: Set up Docker Buildx (needed for imagetools) + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Create & push multi-arch index (GHCR :TAG) via imagetools shell: bash run: | set -euo pipefail - docker manifest create "${DOCKERHUB_IMAGE}:${TAG}" \ - --amend "${DOCKERHUB_IMAGE}:amd64-${TAG}" \ - --amend "${DOCKERHUB_IMAGE}:arm64-${TAG}" - docker manifest push "${DOCKERHUB_IMAGE}:${TAG}" + docker buildx imagetools create \ + -t "${GHCR_IMAGE}:${TAG}" \ + "${GHCR_IMAGE}:amd64-${TAG}" \ + "${GHCR_IMAGE}:arm64-${TAG}" \ + "${GHCR_IMAGE}:armv7-${TAG}" - # Optional tags for non-RC releases: latest, major, minor - - name: Publish additional tags (non-RC only) + - name: Create & push multi-arch index (Docker Hub :TAG) via imagetools + shell: bash + run: | + set -euo pipefail + docker buildx imagetools create \ + -t "${DOCKERHUB_IMAGE}:${TAG}" \ + "${DOCKERHUB_IMAGE}:amd64-${TAG}" \ + "${DOCKERHUB_IMAGE}:arm64-${TAG}" \ + "${DOCKERHUB_IMAGE}:armv7-${TAG}" + + # Additional tags for non-RC releases: latest, major, minor (always) + - name: Publish additional tags (non-RC only) via imagetools if: ${{ env.IS_RC != 'true' }} shell: bash run: | set -euo pipefail - tags_to_publish=() - tags_to_publish+=("${MAJOR_TAG}") - if [ "${PUBLISH_MINOR}" = "true" ]; then - tags_to_publish+=("${MINOR_TAG}") - fi - if [ "${PUBLISH_LATEST}" = "true" ]; then - tags_to_publish+=("latest") - fi + tags_to_publish=("${MAJOR_TAG}" "${MINOR_TAG}" "latest") for t in "${tags_to_publish[@]}"; do echo "Publishing GHCR tag ${t} -> ${TAG}" - docker manifest create "${GHCR_IMAGE}:${t}" \ - --amend "${GHCR_IMAGE}:amd64-${TAG}" \ - --amend "${GHCR_IMAGE}:arm64-${TAG}" - docker manifest push "${GHCR_IMAGE}:${t}" + docker buildx imagetools create \ + -t "${GHCR_IMAGE}:${t}" \ + "${GHCR_IMAGE}:amd64-${TAG}" \ + "${GHCR_IMAGE}:arm64-${TAG}" \ + "${GHCR_IMAGE}:armv7-${TAG}" - if [ -n "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" ]; then - echo "Publishing Docker Hub tag ${t} -> ${TAG}" - docker manifest create "${DOCKERHUB_IMAGE}:${t}" \ - --amend "${DOCKERHUB_IMAGE}:amd64-${TAG}" \ - --amend "${DOCKERHUB_IMAGE}:arm64-${TAG}" - docker manifest push "${DOCKERHUB_IMAGE}:${t}" - fi + echo "Publishing Docker Hub tag ${t} -> ${TAG}" + docker buildx imagetools create \ + -t "${DOCKERHUB_IMAGE}:${t}" \ + "${DOCKERHUB_IMAGE}:amd64-${TAG}" \ + "${DOCKERHUB_IMAGE}:arm64-${TAG}" \ + "${DOCKERHUB_IMAGE}:armv7-${TAG}" done # --------------------------------------------------------------------------- @@ -430,22 +623,29 @@ jobs: name: Sign, attest, and release needs: [create-manifest, build-amd] if: ${{ needs.create-manifest.result == 'success' && needs.build-amd.result == 'success' }} - runs-on: [self-hosted, linux, x64, us-east-1] # NOTE: ensure label exists on runner + runs-on: [self-hosted, linux, x64] # NOTE: ensure label exists on runner timeout-minutes: 120 env: - DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }} + DOCKERHUB_IMAGE: docker.io/${{ github.repository_owner }}/${{ github.event.repository.name }} GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} TAG: ${{ needs.build-amd.outputs.tag }} IS_RC: ${{ needs.build-amd.outputs.is_rc }} + IMAGE_LICENSE: ${{ github.event.repository.license.spdx_id || 'NOASSERTION' }} + IMAGE_CREATED: ${{ needs.pre-run.outputs.image_created }} steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - - name: Capture created timestamp - run: echo "IMAGE_CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV + - name: Ensure repository is at the tagged commit (dispatch only) + if: ${{ github.event_name == 'workflow_dispatch' }} shell: bash + run: | + set -euo pipefail + git fetch --tags --force + git checkout "refs/tags/${TAG}" + echo "Checked out $(git rev-parse --short HEAD) for tag ${TAG}" - name: Install Go uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 @@ -453,7 +653,6 @@ jobs: go-version-file: go.mod - name: Log in to Docker Hub - if: ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_ACCESS_TOKEN != '' }} uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: docker.io @@ -474,26 +673,54 @@ jobs: echo "GHCR_IMAGE=${GHCR_IMAGE,,}" >> "$GITHUB_ENV" echo "DOCKERHUB_IMAGE=${DOCKERHUB_IMAGE,,}" >> "$GITHUB_ENV" + - name: Ensure jq is installed + shell: bash + run: | + set -euo pipefail + if command -v jq >/dev/null 2>&1; then + exit 0 + fi + sudo apt-get update -y + sudo apt-get install -y jq + + - name: Set up Docker Buildx (needed for imagetools) + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + - name: Resolve multi-arch digest refs (by TAG) shell: bash run: | set -euo pipefail - GHCR_DIGEST="$(docker buildx imagetools inspect "${GHCR_IMAGE}:${TAG}" --format '{{.Manifest.Digest}}')" - echo "GHCR_REF=${GHCR_IMAGE}@${GHCR_DIGEST}" >> $GITHUB_ENV - echo "Resolved GHCR_REF=${GHCR_IMAGE}@${GHCR_DIGEST}" - if [ -n "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" ]; then - DH_DIGEST="$(docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:${TAG}" --format '{{.Manifest.Digest}}')" - echo "DH_REF=${DOCKERHUB_IMAGE}@${DH_DIGEST}" >> $GITHUB_ENV - echo "Resolved DH_REF=${DOCKERHUB_IMAGE}@${DH_DIGEST}" - fi - - name: Extract digests for attestation - shell: bash - run: | - set -euo pipefail - echo "GHCR_DIGEST=${GHCR_REF#*@}" >> $GITHUB_ENV - if [ -n "${DH_REF:-}" ]; then - echo "DH_DIGEST=${DH_REF#*@}" >> $GITHUB_ENV + get_digest() { + local ref="$1" + local d="" + # Primary: buildx format output + d="$(docker buildx imagetools inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null || true)" + + # Fallback: parse from plain text if format fails + if ! [[ "$d" =~ ^sha256:[0-9a-f]{64}$ ]]; then + d="$(docker buildx imagetools inspect "$ref" 2>/dev/null | awk '/^Digest:/ {print $2; exit}' || true)" + fi + + if ! [[ "$d" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: Could not extract digest for $ref" >&2 + docker buildx imagetools inspect "$ref" || true + exit 1 + fi + + echo "$d" + } + + GHCR_DIGEST="$(get_digest "${GHCR_IMAGE}:${TAG}")" + echo "GHCR_REF=${GHCR_IMAGE}@${GHCR_DIGEST}" >> "$GITHUB_ENV" + echo "GHCR_DIGEST=${GHCR_DIGEST}" >> "$GITHUB_ENV" + echo "Resolved GHCR_REF=${GHCR_IMAGE}@${GHCR_DIGEST}" + + if [ -n "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" ]; then + DH_DIGEST="$(get_digest "${DOCKERHUB_IMAGE}:${TAG}")" + echo "DH_REF=${DOCKERHUB_IMAGE}@${DH_DIGEST}" >> "$GITHUB_ENV" + echo "DH_DIGEST=${DH_DIGEST}" >> "$GITHUB_ENV" + echo "Resolved DH_REF=${DOCKERHUB_IMAGE}@${DH_DIGEST}" fi - name: Attest build provenance (GHCR) (digest) @@ -509,7 +736,7 @@ jobs: if: ${{ env.DH_DIGEST != '' }} uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 with: - subject-name: index.docker.io/fosrl/${{ github.event.repository.name }} + subject-name: index.docker.io/${{ github.repository_owner }}/${{ github.event.repository.name }} subject-digest: ${{ env.DH_DIGEST }} push-to-registry: true show-summary: true @@ -529,11 +756,12 @@ jobs: cosign public-key --key env://COSIGN_PRIVATE_KEY >/dev/null - name: Generate SBOM (SPDX JSON) from GHCR digest - uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + uses: aquasecurity/trivy-action@97e0b3872f55f89b95b2f65b3dbab56962816478 # v0.34.2 with: image-ref: ${{ env.GHCR_REF }} format: spdx-json output: sbom.spdx.json + version: v0.69.3 - name: Validate + minify SBOM JSON shell: bash @@ -567,6 +795,22 @@ jobs: --predicate sbom.spdx.json \ "${GHCR_REF}" + - name: Create SBOM attestation (Docker Hub, key) + continue-on-error: true + env: + COSIGN_YES: "true" + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + COSIGN_DOCKER_MEDIA_TYPES: "1" + shell: bash + run: | + set -euo pipefail + cosign attest \ + --key env://COSIGN_PRIVATE_KEY \ + --type spdxjson \ + --predicate sbom.spdx.json \ + "${DH_REF}" + - name: Keyless sign & verify GHCR digest (OIDC) env: COSIGN_YES: "true" @@ -596,6 +840,46 @@ jobs: run: cosign verify-attestation --key env://COSIGN_PUBLIC_KEY --type spdxjson "${GHCR_REF}" -o text shell: bash + - name: Sign Docker Hub digest (key, recursive) + continue-on-error: true + env: + COSIGN_YES: "true" + COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + COSIGN_DOCKER_MEDIA_TYPES: "1" + shell: bash + run: | + set -euo pipefail + cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${DH_REF}" + + - name: Keyless sign & verify Docker Hub digest (OIDC) + continue-on-error: true + if: ${{ env.DH_REF != '' }} + env: + COSIGN_YES: "true" + ISSUER: https://token.actions.githubusercontent.com + COSIGN_DOCKER_MEDIA_TYPES: "1" + shell: bash + run: | + set -euo pipefail + cosign sign --rekor-url https://rekor.sigstore.dev --recursive "${DH_REF}" + cosign verify \ + --certificate-oidc-issuer "${ISSUER}" \ + --certificate-identity "https://github.com/${{ github.workflow_ref }}" \ + "${DH_REF}" -o text + + - name: Verify signature (public key) Docker Hub digest + tag + continue-on-error: true + if: ${{ env.DH_REF != '' }} + env: + COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }} + COSIGN_DOCKER_MEDIA_TYPES: "1" + shell: bash + run: | + set -euo pipefail + cosign verify --key env://COSIGN_PUBLIC_KEY "${DH_REF}" -o text + cosign verify --key env://COSIGN_PUBLIC_KEY "${DOCKERHUB_IMAGE}:${TAG}" -o text + - name: Build binaries env: CGO_ENABLED: "0" @@ -624,29 +908,18 @@ jobs: # --------------------------------------------------------------------------- # 7) Stop AWS EC2 runner instances # --------------------------------------------------------------------------- + post-run: name: Stop AWS EC2 runners - needs: [pre-run, prepare, build-amd, build-arm, create-manifest, sign-and-release] - if: >- - ${{ - always() && - (needs.pre-run.result == 'success' && - (needs.build-amd.result == 'success' || needs.build-amd.result == 'failure' || needs.build-amd.result == 'cancelled') || - needs.build-amd.result != 'skipped') && - (needs.build-arm.result == 'success' || needs.build-arm.result == 'failure' || needs.build-arm.result == 'cancelled') || - needs.build-arm.result != 'skipped') && - (needs.create-manifest.result == 'success' || needs.create-manifest.result == 'failure' || needs.create-manifest.result == 'cancelled') || - needs.create-manifest.result != 'skipped') && - (needs.sign-and-release.result == 'success' || needs.sign-and-release.result == 'failure' || needs.sign-and-release.result == 'cancelled') || - needs.sign-and-release.result != 'skipped') - }} + needs: [pre-run, prepare, build-amd, build-arm, build-armv7, create-manifest, sign-and-release] + if: ${{ always() && needs.pre-run.result == 'success' }} runs-on: ubuntu-latest permissions: write-all steps: - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} From fac0f5b1978814bd98b2f8cf994efc79857c27fb Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 7 Mar 2026 10:17:14 -0800 Subject: [PATCH 03/12] Build full arn --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 815c400..4df4ba8 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -70,7 +70,7 @@ jobs: - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} From a85454e77058a1e194b150074a9c9001b7c89d33 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 7 Mar 2026 10:17:14 -0800 Subject: [PATCH 04/12] Build full arn --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 815c400..4df4ba8 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -70,7 +70,7 @@ jobs: - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} From 1bd1133ac2bbf54714de20baec5f4164d444e68a Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 7 Mar 2026 10:36:18 -0800 Subject: [PATCH 05/12] Make sure to skip prepare --- .github/workflows/cicd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4df4ba8..4d55ed4 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -136,7 +136,7 @@ jobs: build-amd: name: Build image (linux/amd64) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, x64] timeout-minutes: 120 env: @@ -293,7 +293,7 @@ jobs: build-arm: name: Build image (linux/arm64) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, arm64] # NOTE: ensure label exists on runner timeout-minutes: 120 env: @@ -417,7 +417,7 @@ jobs: build-armv7: name: Build image (linux/arm/v7) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, arm64] timeout-minutes: 120 env: @@ -919,7 +919,7 @@ jobs: - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} From 392e4c83bfecfb09932c86e22cca8d734fb7d963 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 7 Mar 2026 10:36:18 -0800 Subject: [PATCH 06/12] Make sure to skip prepare --- .github/workflows/cicd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4df4ba8..4d55ed4 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -136,7 +136,7 @@ jobs: build-amd: name: Build image (linux/amd64) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, x64] timeout-minutes: 120 env: @@ -293,7 +293,7 @@ jobs: build-arm: name: Build image (linux/arm64) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, arm64] # NOTE: ensure label exists on runner timeout-minutes: 120 env: @@ -417,7 +417,7 @@ jobs: build-armv7: name: Build image (linux/arm/v7) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && needs.prepare.result == 'success')) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, arm64] timeout-minutes: 120 env: @@ -919,7 +919,7 @@ jobs: - name: Configure AWS credentials (OIDC) uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }} role-duration-seconds: 3600 aws-region: ${{ secrets.AWS_REGION }} From afdb1fc9772f376aefa4db05ed14faf083813176 Mon Sep 17 00:00:00 2001 From: Owen Date: Sat, 7 Mar 2026 12:32:49 -0800 Subject: [PATCH 07/12] Make sure to set version and fix prepare issue --- .github/workflows/cicd.yml | 8 ++++---- Makefile | 23 +++++++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4d55ed4..d0af856 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -136,7 +136,7 @@ jobs: build-amd: name: Build image (linux/amd64) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]' && needs.prepare.result == 'skipped') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, x64] timeout-minutes: 120 env: @@ -293,7 +293,7 @@ jobs: build-arm: name: Build image (linux/arm64) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]' && needs.prepare.result == 'skipped') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, arm64] # NOTE: ensure label exists on runner timeout-minutes: 120 env: @@ -417,7 +417,7 @@ jobs: build-armv7: name: Build image (linux/arm/v7) needs: [pre-run, prepare] - if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} + if: ${{ needs.pre-run.result == 'success' && ((github.event_name == 'push' && github.actor != 'github-actions[bot]' && needs.prepare.result == 'skipped') || (github.event_name == 'workflow_dispatch' && (needs.prepare.result == 'success' || needs.prepare.result == 'skipped'))) }} runs-on: [self-hosted, linux, arm64] timeout-minutes: 120 env: @@ -887,7 +887,7 @@ jobs: shell: bash run: | set -euo pipefail - make -j 10 go-build-release tag="${TAG}" + make -j 10 go-build-release VERSION="${TAG}" - name: Create GitHub Release (draft) uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe # v2.4.2 diff --git a/Makefile b/Makefile index e720189..c35bbbf 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ all: local +VERSION ?= dev +LDFLAGS = -X main.newtVersion=$(VERSION) + local: CGO_ENABLED=0 go build -o ./bin/newt @@ -40,31 +43,31 @@ go-build-release: \ go-build-release-freebsd-arm64 go-build-release-linux-arm64: - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/newt_linux_arm64 + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o bin/newt_linux_arm64 go-build-release-linux-arm32-v7: - CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o bin/newt_linux_arm32 + CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -ldflags "$(LDFLAGS)" -o bin/newt_linux_arm32 go-build-release-linux-arm32-v6: - CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o bin/newt_linux_arm32v6 + CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags "$(LDFLAGS)" -o bin/newt_linux_arm32v6 go-build-release-linux-amd64: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/newt_linux_amd64 + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin/newt_linux_amd64 go-build-release-linux-riscv64: - CGO_ENABLED=0 GOOS=linux GOARCH=riscv64 go build -o bin/newt_linux_riscv64 + CGO_ENABLED=0 GOOS=linux GOARCH=riscv64 go build -ldflags "$(LDFLAGS)" -o bin/newt_linux_riscv64 go-build-release-darwin-arm64: - CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/newt_darwin_arm64 + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o bin/newt_darwin_arm64 go-build-release-darwin-amd64: - CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o bin/newt_darwin_amd64 + CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin/newt_darwin_amd64 go-build-release-windows-amd64: - CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o bin/newt_windows_amd64.exe + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin/newt_windows_amd64.exe go-build-release-freebsd-amd64: - CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -o bin/newt_freebsd_amd64 + CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o bin/newt_freebsd_amd64 go-build-release-freebsd-arm64: - CGO_ENABLED=0 GOOS=freebsd GOARCH=arm64 go build -o bin/newt_freebsd_arm64 + CGO_ENABLED=0 GOOS=freebsd GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o bin/newt_freebsd_arm64 \ No newline at end of file From 768415f90be5b8516eae674a325c8adaebdbfa85 Mon Sep 17 00:00:00 2001 From: Laurence Date: Thu, 5 Mar 2026 15:12:47 +0000 Subject: [PATCH 08/12] Parse target strings with IPv6 support and strict validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add parseTargetString() for listenPort:host:targetPort using net.SplitHostPort/JoinHostPort. Replace manual split in updateTargets; fix err shadowing on remove. Validate listen port 1–65535 and reject empty host/port; use %w for errors. Add tests for IPv4, IPv6, hostnames, and invalid cases. --- common.go | 66 +++++++++++---- common_test.go | 212 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 16 deletions(-) create mode 100644 common_test.go diff --git a/common.go b/common.go index 5fe0645..4701411 100644 --- a/common.go +++ b/common.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "net" "os" "os/exec" "strings" @@ -363,27 +364,62 @@ func parseTargetData(data interface{}) (TargetData, error) { return targetData, nil } +// parseTargetString parses a target string in the format "listenPort:host:targetPort" +// It properly handles IPv6 addresses which must be in brackets: "listenPort:[ipv6]:targetPort" +// Examples: +// - IPv4: "3001:192.168.1.1:80" +// - IPv6: "3001:[::1]:8080" or "3001:[fd70:1452:b736:4dd5:caca:7db9:c588:f5b3]:80" +// +// Returns listenPort, targetAddress (in host:port format suitable for net.Dial), and error +func parseTargetString(target string) (int, string, error) { + // Find the first colon to extract the listen port + firstColon := strings.Index(target, ":") + if firstColon == -1 { + return 0, "", fmt.Errorf("invalid target format, no colon found: %s", target) + } + + listenPortStr := target[:firstColon] + var listenPort int + _, err := fmt.Sscanf(listenPortStr, "%d", &listenPort) + if err != nil { + return 0, "", fmt.Errorf("invalid listen port: %s", listenPortStr) + } + if listenPort <= 0 || listenPort > 65535 { + return 0, "", fmt.Errorf("listen port out of range: %d", listenPort) + } + + // The remainder is host:targetPort - use net.SplitHostPort which handles IPv6 brackets + remainder := target[firstColon+1:] + host, targetPort, err := net.SplitHostPort(remainder) + if err != nil { + return 0, "", fmt.Errorf("invalid host:port format '%s': %w", remainder, err) + } + + // Reject empty host or target port + if host == "" { + return 0, "", fmt.Errorf("empty host in target: %s", target) + } + if targetPort == "" { + return 0, "", fmt.Errorf("empty target port in target: %s", target) + } + + // Reconstruct the target address using JoinHostPort (handles IPv6 properly) + targetAddr := net.JoinHostPort(host, targetPort) + + return listenPort, targetAddr, nil +} + func updateTargets(pm *proxy.ProxyManager, action string, tunnelIP string, proto string, targetData TargetData) error { for _, t := range targetData.Targets { - // Split the first number off of the target with : separator and use as the port - parts := strings.Split(t, ":") - if len(parts) != 3 { - logger.Info("Invalid target format: %s", t) - continue - } - - // Get the port as an int - port := 0 - _, err := fmt.Sscanf(parts[0], "%d", &port) + // Parse the target string, handling both IPv4 and IPv6 addresses + port, target, err := parseTargetString(t) if err != nil { - logger.Info("Invalid port: %s", parts[0]) + logger.Info("Invalid target format: %s (%v)", t, err) continue } switch action { case "add": - target := parts[1] + ":" + parts[2] - // Call updown script if provided processedTarget := target if updownScript != "" { @@ -410,8 +446,6 @@ func updateTargets(pm *proxy.ProxyManager, action string, tunnelIP string, proto case "remove": logger.Info("Removing target with port %d", port) - target := parts[1] + ":" + parts[2] - // Call updown script if provided if updownScript != "" { _, err := executeUpdownScript(action, proto, target) @@ -420,7 +454,7 @@ func updateTargets(pm *proxy.ProxyManager, action string, tunnelIP string, proto } } - err := pm.RemoveTarget(proto, tunnelIP, port) + err = pm.RemoveTarget(proto, tunnelIP, port) if err != nil { logger.Error("Failed to remove target: %v", err) return err diff --git a/common_test.go b/common_test.go new file mode 100644 index 0000000..a7e659a --- /dev/null +++ b/common_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "net" + "testing" +) + +func TestParseTargetString(t *testing.T) { + tests := []struct { + name string + input string + wantListenPort int + wantTargetAddr string + wantErr bool + }{ + // IPv4 test cases + { + name: "valid IPv4 basic", + input: "3001:192.168.1.1:80", + wantListenPort: 3001, + wantTargetAddr: "192.168.1.1:80", + wantErr: false, + }, + { + name: "valid IPv4 localhost", + input: "8080:127.0.0.1:3000", + wantListenPort: 8080, + wantTargetAddr: "127.0.0.1:3000", + wantErr: false, + }, + { + name: "valid IPv4 same ports", + input: "443:10.0.0.1:443", + wantListenPort: 443, + wantTargetAddr: "10.0.0.1:443", + wantErr: false, + }, + + // IPv6 test cases + { + name: "valid IPv6 loopback", + input: "3001:[::1]:8080", + wantListenPort: 3001, + wantTargetAddr: "[::1]:8080", + wantErr: false, + }, + { + name: "valid IPv6 full address", + input: "80:[fd70:1452:b736:4dd5:caca:7db9:c588:f5b3]:8080", + wantListenPort: 80, + wantTargetAddr: "[fd70:1452:b736:4dd5:caca:7db9:c588:f5b3]:8080", + wantErr: false, + }, + { + name: "valid IPv6 link-local", + input: "443:[fe80::1]:443", + wantListenPort: 443, + wantTargetAddr: "[fe80::1]:443", + wantErr: false, + }, + { + name: "valid IPv6 all zeros compressed", + input: "8000:[::]:9000", + wantListenPort: 8000, + wantTargetAddr: "[::]:9000", + wantErr: false, + }, + { + name: "valid IPv6 mixed notation", + input: "5000:[::ffff:192.168.1.1]:6000", + wantListenPort: 5000, + wantTargetAddr: "[::ffff:192.168.1.1]:6000", + wantErr: false, + }, + + // Hostname test cases + { + name: "valid hostname", + input: "8080:example.com:80", + wantListenPort: 8080, + wantTargetAddr: "example.com:80", + wantErr: false, + }, + { + name: "valid hostname with subdomain", + input: "443:api.example.com:8443", + wantListenPort: 443, + wantTargetAddr: "api.example.com:8443", + wantErr: false, + }, + { + name: "valid localhost hostname", + input: "3000:localhost:3000", + wantListenPort: 3000, + wantTargetAddr: "localhost:3000", + wantErr: false, + }, + + // Error cases + { + name: "invalid - no colons", + input: "invalid", + wantErr: true, + }, + { + name: "invalid - empty string", + input: "", + wantErr: true, + }, + { + name: "invalid - non-numeric listen port", + input: "abc:192.168.1.1:80", + wantErr: true, + }, + { + name: "invalid - missing target port", + input: "3001:192.168.1.1", + wantErr: true, + }, + { + name: "invalid - IPv6 without brackets", + input: "3001:fd70:1452:b736:4dd5:caca:7db9:c588:f5b3:80", + wantErr: true, + }, + { + name: "invalid - only listen port", + input: "3001:", + wantErr: true, + }, + { + name: "invalid - missing host", + input: "3001::80", + wantErr: true, + }, + { + name: "invalid - IPv6 unclosed bracket", + input: "3001:[::1:80", + wantErr: true, + }, + { + name: "invalid - listen port zero", + input: "0:192.168.1.1:80", + wantErr: true, + }, + { + name: "invalid - listen port negative", + input: "-1:192.168.1.1:80", + wantErr: true, + }, + { + name: "invalid - listen port out of range", + input: "70000:192.168.1.1:80", + wantErr: true, + }, + { + name: "invalid - empty target port", + input: "3001:192.168.1.1:", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + listenPort, targetAddr, err := parseTargetString(tt.input) + + if (err != nil) != tt.wantErr { + t.Errorf("parseTargetString(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) + return + } + + if tt.wantErr { + return // Don't check other values if we expected an error + } + + if listenPort != tt.wantListenPort { + t.Errorf("parseTargetString(%q) listenPort = %d, want %d", tt.input, listenPort, tt.wantListenPort) + } + + if targetAddr != tt.wantTargetAddr { + t.Errorf("parseTargetString(%q) targetAddr = %q, want %q", tt.input, targetAddr, tt.wantTargetAddr) + } + }) + } +} + +// TestParseTargetStringNetDialCompatibility verifies that the output is compatible with net.Dial +func TestParseTargetStringNetDialCompatibility(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"IPv4", "8080:127.0.0.1:80"}, + {"IPv6 loopback", "8080:[::1]:80"}, + {"IPv6 full", "8080:[2001:db8::1]:80"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, targetAddr, err := parseTargetString(tt.input) + if err != nil { + t.Fatalf("parseTargetString(%q) unexpected error: %v", tt.input, err) + } + + // Verify the format is valid for net.Dial by checking it can be split back + // This doesn't actually dial, just validates the format + _, _, err = net.SplitHostPort(targetAddr) + if err != nil { + t.Errorf("parseTargetString(%q) produced invalid net.Dial format %q: %v", tt.input, targetAddr, err) + } + }) + } +} From accac75a5320c9c03c0143fc136032ae3c63a973 Mon Sep 17 00:00:00 2001 From: Owen Date: Sun, 8 Mar 2026 11:26:22 -0700 Subject: [PATCH 09/12] Set newt version in dockerfile --- .github/workflows/cicd.yml | 3 +++ Dockerfile | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d0af856..3082333 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -269,6 +269,7 @@ jobs: context: . push: true platforms: linux/amd64 + build-args: VERSION=${{ env.TAG }} tags: | ${{ env.GHCR_IMAGE }}:amd64-${{ env.TAG }} ${{ env.DOCKERHUB_IMAGE }}:amd64-${{ env.TAG }} @@ -393,6 +394,7 @@ jobs: context: . push: true platforms: linux/arm64 + build-args: VERSION=${{ env.TAG }} tags: | ${{ env.GHCR_IMAGE }}:arm64-${{ env.TAG }} ${{ env.DOCKERHUB_IMAGE }}:arm64-${{ env.TAG }} @@ -509,6 +511,7 @@ jobs: context: . push: true platforms: linux/arm/v7 + build-args: VERSION=${{ env.TAG }} tags: | ${{ env.GHCR_IMAGE }}:armv7-${{ env.TAG }} ${{ env.DOCKERHUB_IMAGE }}:armv7-${{ env.TAG }} diff --git a/Dockerfile b/Dockerfile index 25113a9..ea870c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,8 @@ RUN go mod download COPY . . # Build the application -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /newt +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X main.newtVersion=${VERSION}" -o /newt FROM public.ecr.aws/docker/library/alpine:3.23 AS runner From d68a13ea1fe50a4f36ac74f52cf093999c1aaf5f Mon Sep 17 00:00:00 2001 From: Laurence Date: Tue, 10 Mar 2026 13:53:39 +0000 Subject: [PATCH 10/12] feat(installer): prefer /usr/local/bin and improve POSIX compatibility - Always install to /usr/local/bin instead of ~/.local/bin - Use sudo automatically when write access is needed - Replace bash-specific syntax with POSIX equivalents: - Change shebang from #!/bin/bash to #!/bin/sh - Replace [[ == *pattern* ]] with case statements - Replace echo -e with printf for colored output - Script now works with dash, ash, busybox sh, and bash --- get-newt.sh | 119 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 73 insertions(+), 46 deletions(-) diff --git a/get-newt.sh b/get-newt.sh index d57f69a..d4ddd3f 100644 --- a/get-newt.sh +++ b/get-newt.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/bin/sh # Get Newt - Cross-platform installation script -# Usage: curl -fsSL https://raw.githubusercontent.com/fosrl/newt/refs/heads/main/get-newt.sh | bash +# Usage: curl -fsSL https://raw.githubusercontent.com/fosrl/newt/refs/heads/main/get-newt.sh | sh set -e @@ -17,15 +17,15 @@ GITHUB_API_URL="https://api.github.com/repos/${REPO}/releases/latest" # Function to print colored output print_status() { - echo -e "${GREEN}[INFO]${NC} $1" + printf '%b[INFO]%b %s\n' "${GREEN}" "${NC}" "$1" } print_warning() { - echo -e "${YELLOW}[WARN]${NC} $1" + printf '%b[WARN]%b %s\n' "${YELLOW}" "${NC}" "$1" } print_error() { - echo -e "${RED}[ERROR]${NC} $1" + printf '%b[ERROR]%b %s\n' "${RED}" "${NC}" "$1" } # Function to get latest version from GitHub API @@ -113,16 +113,34 @@ get_install_dir() { if [ "$OS" = "windows" ]; then echo "$HOME/bin" else - # Try to use a directory in PATH, fallback to ~/.local/bin - if echo "$PATH" | grep -q "/usr/local/bin"; then - if [ -w "/usr/local/bin" ] 2>/dev/null; then - echo "/usr/local/bin" - else - echo "$HOME/.local/bin" - fi + # Prefer /usr/local/bin for system-wide installation + echo "/usr/local/bin" + fi +} + +# Check if we need sudo for installation +needs_sudo() { + local install_dir="$1" + if [ -w "$install_dir" ] 2>/dev/null; then + return 1 # No sudo needed + else + return 0 # Sudo needed + fi +} + +# Get the appropriate command prefix (sudo or empty) +get_sudo_cmd() { + local install_dir="$1" + if needs_sudo "$install_dir"; then + if command -v sudo >/dev/null 2>&1; then + echo "sudo" else - echo "$HOME/.local/bin" + print_error "Cannot write to ${install_dir} and sudo is not available." + print_error "Please run this script as root or install sudo." + exit 1 fi + else + echo "" fi } @@ -130,21 +148,24 @@ get_install_dir() { install_newt() { local platform="$1" local install_dir="$2" + local sudo_cmd="$3" local binary_name="newt_${platform}" local exe_suffix="" - + # Add .exe suffix for Windows - if [[ "$platform" == *"windows"* ]]; then - binary_name="${binary_name}.exe" - exe_suffix=".exe" - fi - + case "$platform" in + *windows*) + binary_name="${binary_name}.exe" + exe_suffix=".exe" + ;; + esac + local download_url="${BASE_URL}/${binary_name}" local temp_file="/tmp/newt${exe_suffix}" local final_path="${install_dir}/newt${exe_suffix}" - + print_status "Downloading newt from ${download_url}" - + # Download the binary if command -v curl >/dev/null 2>&1; then curl -fsSL "$download_url" -o "$temp_file" @@ -154,18 +175,22 @@ install_newt() { print_error "Neither curl nor wget is available. Please install one of them." exit 1 fi - + + # Make executable before moving + chmod +x "$temp_file" + # Create install directory if it doesn't exist - mkdir -p "$install_dir" - - # Move binary to install directory - mv "$temp_file" "$final_path" - - # Make executable (not needed on Windows, but doesn't hurt) - chmod +x "$final_path" - + if [ -n "$sudo_cmd" ]; then + $sudo_cmd mkdir -p "$install_dir" + print_status "Using sudo to install to ${install_dir}" + $sudo_cmd mv "$temp_file" "$final_path" + else + mkdir -p "$install_dir" + mv "$temp_file" "$final_path" + fi + print_status "newt installed to ${final_path}" - + # Check if install directory is in PATH if ! echo "$PATH" | grep -q "$install_dir"; then print_warning "Install directory ${install_dir} is not in your PATH." @@ -179,9 +204,9 @@ verify_installation() { local install_dir="$1" local exe_suffix="" - if [[ "$PLATFORM" == *"windows"* ]]; then - exe_suffix=".exe" - fi + case "$PLATFORM" in + *windows*) exe_suffix=".exe" ;; + esac local newt_path="${install_dir}/newt${exe_suffix}" @@ -198,34 +223,36 @@ verify_installation() { # Main installation process main() { print_status "Installing latest version of newt..." - + # Get latest version print_status "Fetching latest version from GitHub..." VERSION=$(get_latest_version) print_status "Latest version: v${VERSION}" - + # Set base URL with the fetched version BASE_URL="https://github.com/${REPO}/releases/download/${VERSION}" - + # Detect platform PLATFORM=$(detect_platform) print_status "Detected platform: ${PLATFORM}" - + # Get install directory INSTALL_DIR=$(get_install_dir) print_status "Install directory: ${INSTALL_DIR}" - + + # Check if we need sudo + SUDO_CMD=$(get_sudo_cmd "$INSTALL_DIR") + if [ -n "$SUDO_CMD" ]; then + print_status "Root privileges required for installation to ${INSTALL_DIR}" + fi + # Install newt - install_newt "$PLATFORM" "$INSTALL_DIR" - + install_newt "$PLATFORM" "$INSTALL_DIR" "$SUDO_CMD" + # Verify installation if verify_installation "$INSTALL_DIR"; then print_status "newt is ready to use!" - if [[ "$PLATFORM" == *"windows"* ]]; then - print_status "Run 'newt --help' to get started" - else - print_status "Run 'newt --help' to get started" - fi + print_status "Run 'newt --help' to get started" else exit 1 fi From 539e595c4821107b26b8e2ad390b547076bc81df Mon Sep 17 00:00:00 2001 From: Owen Date: Thu, 12 Mar 2026 17:49:05 -0700 Subject: [PATCH 11/12] Add optional compression --- websocket/client.go | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/websocket/client.go b/websocket/client.go index da1fa88..c4daf5f 100644 --- a/websocket/client.go +++ b/websocket/client.go @@ -2,6 +2,7 @@ package websocket import ( "bytes" + "compress/gzip" "crypto/tls" "crypto/x509" "encoding/json" @@ -709,10 +710,13 @@ func (c *Client) readPumpWithDisconnectDetection(started time.Time) { disconnectResult = "success" return default: - var msg WSMessage - err := c.conn.ReadJSON(&msg) + msgType, p, err := c.conn.ReadMessage() if err == nil { - telemetry.IncWSMessage(c.metricsContext(), "in", "text") + if msgType == websocket.BinaryMessage { + telemetry.IncWSMessage(c.metricsContext(), "in", "binary") + } else { + telemetry.IncWSMessage(c.metricsContext(), "in", "text") + } } if err != nil { // Check if we're shutting down before logging error @@ -737,6 +741,29 @@ func (c *Client) readPumpWithDisconnectDetection(started time.Time) { } } + var data []byte + if msgType == websocket.BinaryMessage { + gr, err := gzip.NewReader(bytes.NewReader(p)) + if err != nil { + logger.Error("WebSocket failed to create gzip reader: %v", err) + continue + } + data, err = io.ReadAll(gr) + gr.Close() + if err != nil { + logger.Error("WebSocket failed to decompress message: %v", err) + continue + } + } else { + data = p + } + + var msg WSMessage + if err = json.Unmarshal(data, &msg); err != nil { + logger.Error("WebSocket failed to parse message: %v", err) + continue + } + c.handlersMux.RLock() if handler, ok := c.handlers[msg.Type]; ok { handler(msg) From c7b01288e0ccd0228cb67a4dac2fb087df301b4f Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 13 Mar 2026 11:45:36 -0700 Subject: [PATCH 12/12] Clean up previous logging --- clients/clients.go | 2 -- main.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/clients/clients.go b/clients/clients.go index 05ed3cf..537848d 100644 --- a/clients/clients.go +++ b/clients/clients.go @@ -112,8 +112,6 @@ func NewWireGuardService(interfaceName string, port uint16, mtu int, host string return nil, fmt.Errorf("failed to generate private key: %v", err) } - logger.Debug("+++++++++++++++++++++++++++++++= the port is %d", port) - if port == 0 { // Find an available port portRandom, err := util.FindAvailableUDPPort(49152, 65535) diff --git a/main.go b/main.go index 9c373b0..fc6a890 100644 --- a/main.go +++ b/main.go @@ -619,8 +619,6 @@ func runNewtMain(ctx context.Context) { var wgData WgData var dockerEventMonitor *docker.EventMonitor - logger.Debug("++++++++++++++++++++++ the port is %d", port) - if !disableClients { setupClients(client) }